CoCalc -- codeBlockProcessor.spec.ts

GitHub Repository: microsoft/vscode
Path: blob/main/extensions/copilot/src/extension/codeBlocks/node/test/codeBlockProcessor.spec.ts
¹³⁴⁰³ views
1
/*---------------------------------------------------------------------------------------------
2
 *  Copyright (c) Microsoft Corporation. All rights reserved.
3
 *  Licensed under the MIT License. See License.txt in the project root for license information.
4
 *--------------------------------------------------------------------------------------------*/
5

6
import { assert } from 'chai';
7
import { suite, test } from 'vitest';
8
import type { ChatVulnerability } from 'vscode';
9
import { URI } from '../../../../util/vs/base/common/uri';
10
import { MarkdownString } from '../../../../vscodeTypes';
11
import { CodeBlock } from '../../../prompt/common/conversation';
12
import { CodeBlockInfo, CodeBlockProcessor, LineProcessor } from '../codeBlockProcessor';
13

14
suite('CodeBlockProcessor', () => {
15

16
	type ReportedMarkdown = { markdown: string; codeBlock: CodeBlockInfo | undefined; vulnerabilities: ChatVulnerability[] | undefined };
17

18
	function newCodeBlockProcessor(reportedCodeblocks: CodeBlock[] = [], reportedMarkdown: ReportedMarkdown[] = [], lineProcessor?: LineProcessor) {
19
		return new CodeBlockProcessor(
20
			(path) => URI.file(path),
21
			(markdown, codeBlockInfo, vulnerabilities) => reportedMarkdown.push({ markdown: markdown.value, codeBlock: codeBlockInfo, vulnerabilities }),
22
			(codeblock) => reportedCodeblocks.push(codeblock),
23
			lineProcessor
24
		);
25
	}
26

27
	test('append multi line text', () => {
28
		const reportedCodeblocks: CodeBlock[] = [];
29
		const reportedMarkdown: ReportedMarkdown[] = [];
30
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
31

32
		const lines = [
33
			'hello\n',
34
			'```ts\n',
35
			'console.log("Hello, world!");\n',
36
			'```'
37
		].join('');
38
		tracker.processMarkdown(lines, undefined);
39
		tracker.flush();
40

41
		assert.deepEqual(reportedCodeblocks[0], {
42
			code: 'console.log("Hello, world!");\n',
43
			markdownBeforeBlock: 'hello\n',
44
			language: 'ts',
45
			resource: undefined
46
		});
47

48
		const resource = undefined;
49
		const language = 'ts';
50

51
		assert.deepEqual(reportedMarkdown, [
52
			{ markdown: 'hello\n', codeBlock: undefined, vulnerabilities: undefined },
53
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
54
			{ markdown: 'console.log("Hello, world!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
55
			{ markdown: '```', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
56
		]);
57

58
	});
59

60
	test('append muliple lines', () => {
61
		const reportedCodeblocks: CodeBlock[] = [];
62
		const reportedMarkdown: ReportedMarkdown[] = [];
63

64
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
65
		[
66
			'hello\n',
67
			'```ts\n',
68
			'console.log("Hello!");\n',
69
			'console.log("World!");\n',
70
			'```'
71
		].forEach(line => tracker.processMarkdown(line, undefined));
72
		tracker.flush();
73

74
		const resource = undefined;
75
		const language = 'ts';
76

77
		assert.deepEqual(reportedCodeblocks[0], {
78
			code: 'console.log("Hello!");\nconsole.log("World!");\n',
79
			markdownBeforeBlock: 'hello\n',
80
			language,
81
			resource
82
		});
83

84
		assert.deepEqual(reportedMarkdown, [
85
			{ markdown: 'hello\n', codeBlock: undefined, vulnerabilities: undefined },
86
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
87
			{ markdown: 'console.log("Hello!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
88
			{ markdown: 'console.log("World!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
89
			{ markdown: '```', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
90
		]);
91

92
	});
93

94
	test('append muliple partial lines', () => {
95
		const reportedCodeblocks: CodeBlock[] = [];
96
		const reportedMarkdown: ReportedMarkdown[] = [];
97

98
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
99
		[
100
			'he', 'llo\n',
101
			'```', 'ts\n',
102
			'console', '.log("Hello!");\nconsole',
103
			'.log("World!");\n',
104
			'```'
105
		].forEach(line => tracker.processMarkdown(line, undefined));
106
		tracker.flush();
107

108
		const resource = undefined;
109
		const language = 'ts';
110

111
		assert.deepEqual(reportedCodeblocks[0], {
112
			code: 'console.log("Hello!");\nconsole.log("World!");\n',
113
			markdownBeforeBlock: 'hello\n',
114
			language,
115
			resource
116
		});
117

118
		assert.deepEqual(reportedMarkdown, [
119
			{ markdown: 'he', codeBlock: undefined, vulnerabilities: undefined },
120
			{ markdown: 'llo\n', codeBlock: undefined, vulnerabilities: undefined },
121
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
122
			{ markdown: 'console.log("Hello!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
123
			{ markdown: 'console', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
124
			{ markdown: '.log("World!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
125
			{ markdown: '```', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
126
		]);
127
	});
128

129
	test('append partial lines 1', () => {
130
		const reportedCodeblocks: CodeBlock[] = [];
131
		const reportedMarkdown: ReportedMarkdown[] = [];
132

133
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
134
		[
135
			'12', '345\n',
136
			'`123`', '```456```\n',
137
			'`', '`', '123\n``\n',
138
			'`', '`', '`', 'ts\n',
139
			'# filepath: /project/foo\n',
140
			'`', '`', '123``',
141
			'456\n',
142
			'```'
143
		].forEach(line => tracker.processMarkdown(line, undefined));
144
		tracker.flush();
145

146
		const resource = URI.file('/project/foo');
147
		const language = 'ts';
148

149
		assert.deepEqual(reportedCodeblocks[0], {
150
			code: '``123``456\n',
151
			markdownBeforeBlock: '12345\n`123````456```\n``123\n``\n',
152
			language,
153
			resource
154
		});
155

156
		assert.deepEqual(reportedMarkdown, [
157
			{ markdown: '12', codeBlock: undefined, vulnerabilities: undefined },
158
			{ markdown: '345\n', codeBlock: undefined, vulnerabilities: undefined },
159
			{ markdown: '`123`', codeBlock: undefined, vulnerabilities: undefined },
160
			{ markdown: '```456```\n', codeBlock: undefined, vulnerabilities: undefined },
161
			{ markdown: '``123\n', codeBlock: undefined, vulnerabilities: undefined },
162
			{ markdown: '``\n', codeBlock: undefined, vulnerabilities: undefined },
163
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
164
			{ markdown: '``123``456\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
165
			{ markdown: '```', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
166
		]);
167
	});
168

169

170
	test('multiple code blocks', () => {
171
		const reportedCodeblocks: CodeBlock[] = [];
172
		const reportedMarkdown: ReportedMarkdown[] = [];
173

174
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
175
		tracker.processMarkdown([
176
			'hello\n',
177
			'```ts\n',
178
			'console.log("Hello, world!");\n',
179
			'```\n',
180
			'more\n',
181
			'more\n',
182
			'more\n',
183
			'```ts\n',
184
			'console.log("more");\n',
185
			'```'
186
		].join(''));
187
		tracker.flush();
188

189
		const language = 'ts';
190
		const resource = undefined;
191

192
		assert.deepEqual(reportedCodeblocks[0], {
193
			code: 'console.log("Hello, world!");\n',
194
			markdownBeforeBlock: 'hello\n',
195
			language,
196
			resource
197
		});
198
		assert.deepEqual(reportedCodeblocks[1], {
199
			code: 'console.log("more");\n',
200
			markdownBeforeBlock: 'more\nmore\nmore\n',
201
			language,
202
			resource
203
		});
204

205

206
		assert.deepEqual(reportedMarkdown, [
207
			{ markdown: 'hello\n', codeBlock: undefined, vulnerabilities: undefined },
208
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
209
			{ markdown: 'console.log("Hello, world!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
210
			{ markdown: '```\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
211
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
212
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
213
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
214
			{ markdown: '```ts\n', codeBlock: { index: 1, resource, language }, vulnerabilities: undefined },
215
			{ markdown: 'console.log("more");\n', codeBlock: { index: 1, resource, language }, vulnerabilities: undefined },
216
			{ markdown: '```', codeBlock: { index: 1, resource, language }, vulnerabilities: undefined },
217
		]);
218

219

220
	});
221

222
	test('code blocks with tildes', () => {
223
		const reportedCodeblocks: CodeBlock[] = [];
224
		const reportedMarkdown: ReportedMarkdown[] = [];
225

226
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
227

228
		tracker.processMarkdown([
229
			'~~~ts\n',
230
			'// using tilde\n',
231
			'~~~\n',
232
			'````ts\n',
233
			'// using 4 backticks\n',
234
			'````\n',
235
		].join(''));
236
		tracker.flush();
237

238
		const resource = undefined;
239
		const language = 'ts';
240

241
		assert.deepEqual(reportedCodeblocks[0], {
242
			code: '// using tilde\n',
243
			markdownBeforeBlock: '',
244
			language,
245
			resource
246
		});
247
		assert.deepEqual(reportedCodeblocks[1], {
248
			code: '// using 4 backticks\n',
249
			markdownBeforeBlock: '',
250
			language,
251
			resource
252
		});
253

254
		assert.deepEqual(reportedMarkdown, [
255
			{ markdown: '~~~ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
256
			{ markdown: '// using tilde\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
257
			{ markdown: '~~~\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
258
			{ markdown: '````ts\n', codeBlock: { index: 1, resource, language }, vulnerabilities: undefined },
259
			{ markdown: '// using 4 backticks\n', codeBlock: { index: 1, resource, language }, vulnerabilities: undefined },
260
			{ markdown: '````\n', codeBlock: { index: 1, resource, language }, vulnerabilities: undefined },
261
		]);
262
	});
263

264
	test('nested code blocks', () => {
265
		const reportedCodeblocks: CodeBlock[] = [];
266
		const reportedMarkdown: ReportedMarkdown[] = [];
267

268
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
269
		tracker.processMarkdown([
270
			'````ts\n',
271
			'// using 4 backticks\n',
272
			'```ts\n',
273
			'// nested using 3 backticks\n',
274
			'```\n',
275
			'````\n',
276
		].join(''));
277
		tracker.flush();
278

279
		const resource = undefined;
280
		const language = 'ts';
281

282
		assert.deepEqual(reportedCodeblocks[0], {
283
			code: [
284
				'// using 4 backticks\n',
285
				'```ts\n',
286
				'// nested using 3 backticks\n',
287
				'```\n',
288
			].join(''),
289
			markdownBeforeBlock: '',
290
			language,
291
			resource
292
		});
293

294
		assert.deepEqual(reportedMarkdown, [
295
			{ markdown: '````ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
296
			{ markdown: '// using 4 backticks\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
297
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
298
			{ markdown: '// nested using 3 backticks\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
299
			{ markdown: '```\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
300
			{ markdown: '````\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
301
		]);
302
	});
303

304
	test('file marker', () => {
305
		const reportedCodeblocks: CodeBlock[] = [];
306
		const reportedMarkdown: ReportedMarkdown[] = [];
307

308
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
309
		tracker.processMarkdown([
310
			'hello\n',
311
			'```ts\n',
312
			'// filepath: /project/foo0\n',
313
			'console.log("Hello, world!");\n',
314
			'```\n',
315
			'more\n',
316
			'more\n',
317
			'more\n',
318
			'```html\n',
319
			'<!-- filepath: /project/foo1 -->\n',
320
			'<html>more</html>\n',
321
			'```'
322
		].join(''));
323
		tracker.flush();
324

325
		const resource0 = URI.file('/project/foo0');
326
		const language0 = 'ts';
327
		const resource1 = URI.file('/project/foo1');
328
		const language1 = 'html';
329

330
		assert.deepEqual(reportedCodeblocks[0], {
331
			code: 'console.log("Hello, world!");\n',
332
			markdownBeforeBlock: 'hello\n',
333
			language: language0,
334
			resource: resource0
335
		});
336
		assert.deepEqual(reportedCodeblocks[1], {
337
			code: '<html>more</html>\n',
338
			markdownBeforeBlock: 'more\nmore\nmore\n',
339
			language: language1,
340
			resource: resource1
341
		});
342

343

344

345
		assert.deepEqual(reportedMarkdown, [
346
			{ markdown: 'hello\n', codeBlock: undefined, vulnerabilities: undefined },
347
			{ markdown: '```ts\n', codeBlock: { index: 0, resource: resource0, language: language0 }, vulnerabilities: undefined },
348
			{ markdown: 'console.log("Hello, world!");\n', codeBlock: { index: 0, resource: resource0, language: language0 }, vulnerabilities: undefined },
349
			{ markdown: '```\n', codeBlock: { index: 0, resource: resource0, language: language0 }, vulnerabilities: undefined },
350
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
351
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
352
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
353
			{ markdown: '```html\n', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
354
			{ markdown: '<html>more</html>\n', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
355
			{ markdown: '```', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
356
		]);
357
	});
358

359
	test('new line after file marker', () => {
360
		const reportedCodeblocks: CodeBlock[] = [];
361
		const reportedMarkdown: ReportedMarkdown[] = [];
362

363
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
364
		tracker.processMarkdown([
365
			'hello\n',
366
			'```ts\n',
367
			'// filepath: /project/foo0\n',
368
			'\n',
369
			'console.log("Hello, world!");\n',
370
			'```\n',
371
			'more\n',
372
			'more\n',
373
			'more\n',
374
			'```html\n',
375
			'<!-- filepath: /project/foo1 -->\n',
376
			'\n',
377
			'\n',
378
			'<html>more</html>\n',
379
			'```'
380
		].join(''));
381
		tracker.flush();
382

383
		const resource0 = URI.file('/project/foo0');
384
		const language0 = 'ts';
385
		const resource1 = URI.file('/project/foo1');
386
		const language1 = 'html';
387

388
		assert.deepEqual(reportedCodeblocks[0], {
389
			code: 'console.log("Hello, world!");\n',
390
			markdownBeforeBlock: 'hello\n',
391
			language: language0,
392
			resource: resource0
393
		});
394
		assert.deepEqual(reportedCodeblocks[1], {
395
			code: '\n<html>more</html>\n',
396
			markdownBeforeBlock: 'more\nmore\nmore\n',
397
			language: language1,
398
			resource: resource1
399
		});
400

401
		assert.deepEqual(reportedMarkdown, [
402
			{ markdown: 'hello\n', codeBlock: undefined, vulnerabilities: undefined },
403
			{ markdown: '```ts\n', codeBlock: { index: 0, resource: resource0, language: language0 }, vulnerabilities: undefined },
404
			{ markdown: 'console.log("Hello, world!");\n', codeBlock: { index: 0, resource: resource0, language: language0 }, vulnerabilities: undefined },
405
			{ markdown: '```\n', codeBlock: { index: 0, resource: resource0, language: language0 }, vulnerabilities: undefined },
406
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
407
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
408
			{ markdown: 'more\n', codeBlock: undefined, vulnerabilities: undefined },
409
			{ markdown: '```html\n', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
410
			{ markdown: '\n', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
411
			{ markdown: '<html>more</html>\n', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
412
			{ markdown: '```', codeBlock: { index: 1, resource: resource1, language: language1 }, vulnerabilities: undefined },
413
		]);
414
	});
415

416

417
	test('file marker reported', () => {
418
		const reportedCodeblocks: CodeBlock[] = [];
419
		const reportedMarkdown: ReportedMarkdown[] = [];
420

421
		const resource = URI.file('/project/foo');
422

423
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
424
		tracker.processMarkdown([
425
			'hello\n',
426
			'```ts\n',
427
		].join(''));
428
		tracker.processCodeblockUri(resource);
429
		tracker.processMarkdown([
430
			'console.log("Hello, world!");\n',
431
			'```\n',
432
		].join(''));
433
		tracker.flush();
434

435
		const language = 'ts';
436

437
		assert.deepEqual(reportedCodeblocks[0], {
438
			code: 'console.log("Hello, world!");\n',
439
			markdownBeforeBlock: 'hello\n',
440
			language: language,
441
			resource: resource
442
		});
443

444
		assert.deepEqual(reportedMarkdown, [
445
			{ markdown: 'hello\n', codeBlock: undefined, vulnerabilities: undefined },
446
			{ markdown: '```ts\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
447
			{ markdown: 'console.log("Hello, world!");\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
448
			{ markdown: '```\n', codeBlock: { index: 0, resource, language }, vulnerabilities: undefined },
449
		]);
450

451
	});
452

453
	test('nested codeblocks with the same separator', () => {
454
		const reportedCodeblocks: CodeBlock[] = [];
455
		const reportedMarkdown: ReportedMarkdown[] = [];
456

457
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown);
458
		const lines = [
459
			'```markdown\n',
460
			'# Example Markdown Document\n',
461
			'\n',
462
			'This is an example of a Markdown document that contains a code block.\n',
463
			'\n',
464
			'## Code Block\n',
465
			'\n',
466
			'Here is a code block in TypeScript:\n',
467
			'\n',
468
			'```typescript\n',
469
			'// Generated by Copilot\n',
470
			'class Example {\n',
471
			'    private _value: number;\n',
472
			'}\n',
473
			'```\n',
474
			'```\n'
475
		];
476
		tracker.processMarkdown(lines.join(''));
477
		tracker.flush();
478

479
		const resource = undefined;
480
		const language = 'markdown';
481

482
		assert.deepEqual(reportedCodeblocks[0], {
483
			code: lines.slice(1, lines.length - 1).join(''),
484
			markdownBeforeBlock: '',
485
			language: language,
486
			resource: resource
487
		});
488

489
		assert.deepEqual(reportedMarkdown, lines.map(markdown => ({ markdown, codeBlock: { index: 0, resource, language }, vulnerabilities: undefined })));
490

491

492
	});
493

494
	test('line handler', () => {
495
		const reportedCodeblocks: CodeBlock[] = [];
496
		const reportedMarkdown: ReportedMarkdown[] = [];
497

498
		const lineProcessor = {
499
			matchesLineStart(linePart: string, inCodeBlock: boolean): boolean {
500
				return linePart.startsWith('###'.substring(0, linePart.length));
501
			},
502
			process(line: MarkdownString, inCodeBlock: boolean): MarkdownString {
503
				return new MarkdownString(inCodeBlock ? line.value.toLowerCase() : line.value.toUpperCase());
504
			}
505
		};
506

507
		const tracker = newCodeBlockProcessor(reportedCodeblocks, reportedMarkdown, lineProcessor);
508
		const lines = [
509
			'# Big Header\n',
510
			'### Example Header\n',
511
			'\n',
512
			'This is an example of a Markdown document that contains a code block.\n',
513
			'\n',
514
			'#### Outside\n',
515
			'\n',
516
			'Here is a code block:\n',
517
			'\n',
518
			'```markdown\n', // line 9
519
			'# Unrelated\n',
520
			'## Unrelated\n',
521
			'### Inside\n',
522
			'```\n',
523
		];
524
		// process character by character to simulate streaming
525
		lines.join('').split('').forEach(s =>
526
			tracker.processMarkdown(s)
527
		);
528
		tracker.flush();
529

530
		const resource = undefined;
531
		const language = 'markdown';
532

533
		const expectedLines = [...lines];
534
		expectedLines[1] = '### EXAMPLE HEADER\n';
535
		expectedLines[5] = '#### OUTSIDE\n';
536
		expectedLines[12] = '### inside\n';
537

538

539
		assert.deepEqual(reportedCodeblocks[0], {
540
			code: expectedLines.slice(10, 13).join(''),
541
			markdownBeforeBlock: expectedLines.slice(0, 9).join(''),
542
			language: language,
543
			resource: resource
544
		});
545

546
		assert.deepEqual(reportedMarkdown.map(m => m.markdown).join(''), expectedLines.join(''));
547

548

549
	});
550

551
});
552

553
Product

Resources

Company
