CoCalc -- operators

GitHub Repository: projectdiscovery/nuclei
Path: blob/dev/pkg/protocols/headless/operators_test.go
²⁸⁴⁶ views
1
package headless
2

3
import (
4
	"testing"
5

6
	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
7
	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
8
	"github.com/stretchr/testify/require"
9
)
10

11
func TestRequest_ExtractXPath(t *testing.T) {
12
	request := &Request{}
13

14
	// Test HTML content extraction
15
	htmlContent := `<!doctype html>
16
<html>
17
<head>
18
    <title>Test Page</title>
19
</head>
20
<body>
21
    <div class="container">
22
        <h1>Welcome</h1>
23
        <p>This is a test page</p>
24
        <a href="https://example.com" id="test-link">Click here</a>
25
        <ul>
26
            <li>Item 1</li>
27
            <li>Item 2</li>
28
            <li>Item 3</li>
29
        </ul>
30
    </div>
31
</body>
32
</html>`
33

34
	data := map[string]interface{}{
35
		"data": htmlContent,
36
	}
37

38
	// Test extracting text content
39
	extractor := &extractors.Extractor{
40
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
41
		XPath: []string{"/html/body/div/h1"},
42
	}
43
	err := extractor.CompileExtractors()
44
	require.Nil(t, err)
45

46
	result := request.Extract(data, extractor)
47
	expected := map[string]struct{}{"Welcome": {}}
48
	require.Equal(t, expected, result)
49

50
	// Test extracting attribute value
51
	extractor = &extractors.Extractor{
52
		Type:      extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
53
		XPath:     []string{"/html/body/div/a"},
54
		Attribute: "href",
55
	}
56
	err = extractor.CompileExtractors()
57
	require.Nil(t, err)
58

59
	result = request.Extract(data, extractor)
60
	expected = map[string]struct{}{"https://example.com": {}}
61
	require.Equal(t, expected, result)
62

63
	// Test extracting multiple items
64
	extractor = &extractors.Extractor{
65
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
66
		XPath: []string{"/html/body/div/ul/li"},
67
	}
68
	err = extractor.CompileExtractors()
69
	require.Nil(t, err)
70

71
	result = request.Extract(data, extractor)
72
	expected = map[string]struct{}{
73
		"Item 1": {},
74
		"Item 2": {},
75
		"Item 3": {},
76
	}
77
	require.Equal(t, expected, result)
78

79
	// Test with non-existent XPath
80
	extractor = &extractors.Extractor{
81
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
82
		XPath: []string{"/html/body/div/nonexistent"},
83
	}
84
	err = extractor.CompileExtractors()
85
	require.Nil(t, err)
86

87
	result = request.Extract(data, extractor)
88
	require.Equal(t, map[string]struct{}{}, result)
89
}
90

91
func TestRequest_ExtractJSON(t *testing.T) {
92
	request := &Request{}
93

94
	// Test JSON content extraction
95
	jsonContent := `{
96
		"users": [
97
			{"id": 1, "name": "John", "email": "[email protected]"},
98
			{"id": 2, "name": "Jane", "email": "[email protected]"},
99
			{"id": 3, "name": "Bob", "email": "[email protected]"}
100
		],
101
		"metadata": {
102
			"total": 3,
103
			"page": 1
104
		}
105
	}`
106

107
	data := map[string]interface{}{
108
		"data": jsonContent,
109
	}
110

111
	// Test extracting user IDs
112
	extractor := &extractors.Extractor{
113
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
114
		JSON: []string{".users[].id"},
115
	}
116
	err := extractor.CompileExtractors()
117
	require.Nil(t, err)
118

119
	result := request.Extract(data, extractor)
120
	expected := map[string]struct{}{
121
		"1": {},
122
		"2": {},
123
		"3": {},
124
	}
125
	require.Equal(t, expected, result)
126

127
	// Test extracting user names
128
	extractor = &extractors.Extractor{
129
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
130
		JSON: []string{".users[].name"},
131
	}
132
	err = extractor.CompileExtractors()
133
	require.Nil(t, err)
134

135
	result = request.Extract(data, extractor)
136
	expected = map[string]struct{}{
137
		"John": {},
138
		"Jane": {},
139
		"Bob":  {},
140
	}
141
	require.Equal(t, expected, result)
142

143
	// Test extracting nested values
144
	extractor = &extractors.Extractor{
145
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
146
		JSON: []string{".metadata.total"},
147
	}
148
	err = extractor.CompileExtractors()
149
	require.Nil(t, err)
150

151
	result = request.Extract(data, extractor)
152
	expected = map[string]struct{}{"3": {}}
153
	require.Equal(t, expected, result)
154

155
	// Test extracting emails
156
	extractor = &extractors.Extractor{
157
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
158
		JSON: []string{".users[].email"},
159
	}
160
	err = extractor.CompileExtractors()
161
	require.Nil(t, err)
162

163
	result = request.Extract(data, extractor)
164
	expected = map[string]struct{}{
165
		"[email protected]": {},
166
		"[email protected]": {},
167
		"[email protected]":  {},
168
	}
169
	require.Equal(t, expected, result)
170

171
	// Test with invalid JSON
172
	invalidJSON := `{"invalid": json}`
173
	data = map[string]interface{}{
174
		"data": invalidJSON,
175
	}
176

177
	extractor = &extractors.Extractor{
178
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
179
		JSON: []string{".invalid"},
180
	}
181
	err = extractor.CompileExtractors()
182
	require.Nil(t, err)
183

184
	result = request.Extract(data, extractor)
185
	require.Equal(t, map[string]struct{}{}, result)
186

187
	// Test with non-existent path
188
	extractor = &extractors.Extractor{
189
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
190
		JSON: []string{".nonexistent"},
191
	}
192
	err = extractor.CompileExtractors()
193
	require.Nil(t, err)
194

195
	result = request.Extract(data, extractor)
196
	require.Equal(t, map[string]struct{}{}, result)
197
}
198

199
func TestRequest_MatchXPath(t *testing.T) {
200
	request := &Request{}
201

202
	htmlContent := `<!doctype html>
203
<html>
204
<head>
205
    <title>Test Page</title>
206
</head>
207
<body>
208
    <div class="container">
209
        <h1>Welcome</h1>
210
        <p>This is a test page</p>
211
        <a href="https://example.com" id="test-link">Click here</a>
212
    </div>
213
</body>
214
</html>`
215

216
	data := map[string]interface{}{
217
		"data": htmlContent,
218
	}
219

220
	// Test XPath matcher with existing element
221
	matcher := &matchers.Matcher{
222
		Type:      matchers.MatcherTypeHolder{MatcherType: matchers.XPathMatcher},
223
		XPath:     []string{"/html/body/div/h1"},
224
		Condition: "and",
225
	}
226
	err := matcher.CompileMatchers()
227
	require.Nil(t, err)
228

229
	matched, snippets := request.Match(data, matcher)
230
	require.True(t, matched)
231
	require.Empty(t, snippets)
232

233
	// Test XPath matcher with non-existent element
234
	matcher = &matchers.Matcher{
235
		Type:      matchers.MatcherTypeHolder{MatcherType: matchers.XPathMatcher},
236
		XPath:     []string{"/html/body/div/nonexistent"},
237
		Condition: "and",
238
	}
239
	err = matcher.CompileMatchers()
240
	require.Nil(t, err)
241

242
	matched, snippets = request.Match(data, matcher)
243
	require.False(t, matched)
244
	require.Empty(t, snippets)
245
}
246

247
func TestRequest_getMatchPart(t *testing.T) {
248
	request := &Request{}
249

250
	data := map[string]interface{}{
251
		"data":    "body content",
252
		"header":  "header content",
253
		"history": "history content",
254
	}
255

256
	// Test default part (should map to "data")
257
	part, ok := request.getMatchPart("", data)
258
	require.True(t, ok)
259
	require.Equal(t, "body content", part)
260

261
	// Test "body" part (should map to "data")
262
	part, ok = request.getMatchPart("body", data)
263
	require.True(t, ok)
264
	require.Equal(t, "body content", part)
265

266
	// Test "resp" part (should map to "data")
267
	part, ok = request.getMatchPart("resp", data)
268
	require.True(t, ok)
269
	require.Equal(t, "body content", part)
270

271
	// Test "header" part
272
	part, ok = request.getMatchPart("header", data)
273
	require.True(t, ok)
274
	require.Equal(t, "header content", part)
275

276
	// Test "history" part
277
	part, ok = request.getMatchPart("history", data)
278
	require.True(t, ok)
279
	require.Equal(t, "history content", part)
280

281
	// Test non-existent part
282
	part, ok = request.getMatchPart("nonexistent", data)
283
	require.False(t, ok)
284
	require.Equal(t, "", part)
285
}
286

287
func TestRequest_ExtractWithDifferentParts(t *testing.T) {
288
	request := &Request{}
289

290
	// Test extracting from different parts
291
	htmlContent := `<!doctype html><html><body><div><h1>Title</h1></div></body></html>`
292
	jsonContent := `{"id": 123}`
293

294
	data := map[string]interface{}{
295
		"data":    htmlContent,
296
		"header":  jsonContent,
297
		"history": htmlContent,
298
	}
299

300
	// Test XPath extractor from "data" part
301
	extractor := &extractors.Extractor{
302
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
303
		XPath: []string{"/html/body/div/h1"},
304
		Part:  "data",
305
	}
306
	err := extractor.CompileExtractors()
307
	require.Nil(t, err)
308

309
	result := request.Extract(data, extractor)
310
	expected := map[string]struct{}{"Title": {}}
311
	require.Equal(t, expected, result)
312

313
	// Test JSON extractor from "header" part
314
	extractor = &extractors.Extractor{
315
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
316
		JSON: []string{".id"},
317
		Part: "header",
318
	}
319
	err = extractor.CompileExtractors()
320
	require.Nil(t, err)
321

322
	result = request.Extract(data, extractor)
323
	expected = map[string]struct{}{"123": {}}
324
	require.Equal(t, expected, result)
325

326
	// Test XPath extractor from "history" part
327
	extractor = &extractors.Extractor{
328
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
329
		XPath: []string{"/html/body/div/h1"},
330
		Part:  "history",
331
	}
332
	err = extractor.CompileExtractors()
333
	require.Nil(t, err)
334

335
	result = request.Extract(data, extractor)
336
	expected = map[string]struct{}{"Title": {}}
337
	require.Equal(t, expected, result)
338
}
339

340
func TestRequest_ExtractWithComplexJSON(t *testing.T) {
341
	request := &Request{}
342

343
	// Test with complex nested JSON structure
344
	jsonContent := `{
345
		"api": {
346
			"version": "1.0",
347
			"endpoints": [
348
				{
349
					"path": "/users",
350
					"method": "GET",
351
					"responses": [
352
						{"code": 200, "description": "Success"},
353
						{"code": 404, "description": "Not Found"}
354
					]
355
				},
356
				{
357
					"path": "/posts",
358
					"method": "POST",
359
					"responses": [
360
						{"code": 201, "description": "Created"},
361
						{"code": 400, "description": "Bad Request"}
362
					]
363
				}
364
			]
365
		}
366
	}`
367

368
	data := map[string]interface{}{
369
		"data": jsonContent,
370
	}
371

372
	// Test extracting API version
373
	extractor := &extractors.Extractor{
374
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
375
		JSON: []string{".api.version"},
376
	}
377
	err := extractor.CompileExtractors()
378
	require.Nil(t, err)
379

380
	result := request.Extract(data, extractor)
381
	expected := map[string]struct{}{"1.0": {}}
382
	require.Equal(t, expected, result)
383

384
	// Test extracting all endpoint paths
385
	extractor = &extractors.Extractor{
386
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
387
		JSON: []string{".api.endpoints[].path"},
388
	}
389
	err = extractor.CompileExtractors()
390
	require.Nil(t, err)
391

392
	result = request.Extract(data, extractor)
393
	expected = map[string]struct{}{
394
		"/users": {},
395
		"/posts": {},
396
	}
397
	require.Equal(t, expected, result)
398

399
	// Test extracting all response codes
400
	extractor = &extractors.Extractor{
401
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
402
		JSON: []string{".api.endpoints[].responses[].code"},
403
	}
404
	err = extractor.CompileExtractors()
405
	require.Nil(t, err)
406

407
	result = request.Extract(data, extractor)
408
	expected = map[string]struct{}{
409
		"200": {},
410
		"404": {},
411
		"201": {},
412
		"400": {},
413
	}
414
	require.Equal(t, expected, result)
415

416
	// Test extracting response descriptions
417
	extractor = &extractors.Extractor{
418
		Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
419
		JSON: []string{".api.endpoints[].responses[].description"},
420
	}
421
	err = extractor.CompileExtractors()
422
	require.Nil(t, err)
423

424
	result = request.Extract(data, extractor)
425
	expected = map[string]struct{}{
426
		"Success":     {},
427
		"Not Found":   {},
428
		"Created":     {},
429
		"Bad Request": {},
430
	}
431
	require.Equal(t, expected, result)
432
}
433

434
func TestRequest_ExtractWithComplexHTML(t *testing.T) {
435
	request := &Request{}
436

437
	// Test with complex HTML structure
438
	htmlContent := `<!doctype html>
439
<html>
440
<head>
441
    <title>E-commerce Site</title>
442
    <meta name="description" content="Online shopping platform">
443
</head>
444
<body>
445
    <header>
446
        <nav>
447
            <ul class="nav-menu">
448
                <li><a href="/home">Home</a></li>
449
                <li><a href="/products">Products</a></li>
450
                <li><a href="/about">About</a></li>
451
            </ul>
452
        </nav>
453
    </header>
454
    <main>
455
        <section class="products">
456
            <h2>Featured Products</h2>
457
            <div class="product-grid">
458
                <div class="product" data-id="1">
459
                    <h3>Laptop</h3>
460
                    <p class="price">$999</p>
461
                    <span class="rating">4.5</span>
462
                </div>
463
                <div class="product" data-id="2">
464
                    <h3>Phone</h3>
465
                    <p class="price">$599</p>
466
                    <span class="rating">4.2</span>
467
                </div>
468
                <div class="product" data-id="3">
469
                    <h3>Tablet</h3>
470
                    <p class="price">$399</p>
471
                    <span class="rating">4.0</span>
472
                </div>
473
            </div>
474
        </section>
475
    </main>
476
    <footer>
477
        <p>&copy; 2024 E-commerce Site</p>
478
    </footer>
479
</body>
480
</html>`
481

482
	data := map[string]interface{}{
483
		"data": htmlContent,
484
	}
485

486
	// Test extracting navigation links
487
	extractor := &extractors.Extractor{
488
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
489
		XPath: []string{"/html/body/header/nav/ul/li/a"},
490
	}
491
	err := extractor.CompileExtractors()
492
	require.Nil(t, err)
493

494
	result := request.Extract(data, extractor)
495
	expected := map[string]struct{}{
496
		"Home":     {},
497
		"Products": {},
498
		"About":    {},
499
	}
500
	require.Equal(t, expected, result)
501

502
	// Test extracting product names
503
	extractor = &extractors.Extractor{
504
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
505
		XPath: []string{"/html/body/main/section/div/div/h3"},
506
	}
507
	err = extractor.CompileExtractors()
508
	require.Nil(t, err)
509

510
	result = request.Extract(data, extractor)
511
	expected = map[string]struct{}{
512
		"Laptop": {},
513
		"Phone":  {},
514
		"Tablet": {},
515
	}
516
	require.Equal(t, expected, result)
517

518
	// Test extracting product prices
519
	extractor = &extractors.Extractor{
520
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
521
		XPath: []string{"/html/body/main/section/div/div/p[@class='price']"},
522
	}
523
	err = extractor.CompileExtractors()
524
	require.Nil(t, err)
525

526
	result = request.Extract(data, extractor)
527
	expected = map[string]struct{}{
528
		"$999": {},
529
		"$599": {},
530
		"$399": {},
531
	}
532
	require.Equal(t, expected, result)
533

534
	// Test extracting product ratings
535
	extractor = &extractors.Extractor{
536
		Type:  extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
537
		XPath: []string{"/html/body/main/section/div/div/span[@class='rating']"},
538
	}
539
	err = extractor.CompileExtractors()
540
	require.Nil(t, err)
541

542
	result = request.Extract(data, extractor)
543
	expected = map[string]struct{}{
544
		"4.5": {},
545
		"4.2": {},
546
		"4.0": {},
547
	}
548
	require.Equal(t, expected, result)
549

550
	// Test extracting data attributes
551
	extractor = &extractors.Extractor{
552
		Type:      extractors.ExtractorTypeHolder{ExtractorType: extractors.XPathExtractor},
553
		XPath:     []string{"/html/body/main/section/div/div[@class='product']"},
554
		Attribute: "data-id",
555
	}
556
	err = extractor.CompileExtractors()
557
	require.Nil(t, err)
558

559
	result = request.Extract(data, extractor)
560
	expected = map[string]struct{}{
561
		"1": {},
562
		"2": {},
563
		"3": {},
564
	}
565
	require.Equal(t, expected, result)
566
}
567

568
Product

Resources

Company
