Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
microsoft
GitHub Repository: microsoft/vscode
Path: blob/main/scripts/chat-simulation/common/perf-scenarios.js
13383 views
1
/*---------------------------------------------------------------------------------------------
2
* Copyright (c) Microsoft Corporation. All rights reserved.
3
* Licensed under the MIT License. See License.txt in the project root for license information.
4
*--------------------------------------------------------------------------------------------*/
5
6
// @ts-check
7
8
/**
9
* Built-in scenario definitions for chat performance benchmarks and leak checks.
10
*
11
* Each test file imports this module and calls `registerScenario()` for the
12
* scenarios it needs, keeping scenario ownership close to the test that uses it.
13
*/
14
15
const path = require('path');
16
const { ScenarioBuilder, registerScenario } = require('./mock-llm-server');
17
18
const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures');
19
20
/**
21
* @typedef {{
22
* description: string,
23
* chunks: import('./mock-llm-server').StreamChunk[],
24
* }} ContentScenarioDef
25
*
26
* @typedef {{
27
* description: string,
28
* scenario: import('./mock-llm-server').MultiTurnScenario,
29
* }} MultiTurnScenarioDef
30
*/
31
32
// -- Content-only scenarios ---------------------------------------------------
33
34
/** @type {Record<string, ContentScenarioDef>} */
35
const CONTENT_SCENARIOS = {
36
'text-only': {
37
description: 'Plain text, 4 paragraphs',
38
chunks: new ScenarioBuilder()
39
.stream([
40
'Here is an explanation of the code you selected:\n\n',
41
'The function `processItems` iterates over the input array and applies a transformation to each element. ',
42
'It uses a `Map` to track previously seen values, which allows it to deduplicate results efficiently in O(n) time.\n\n',
43
'The algorithm works in a single pass: for every element, it computes the transformed value, ',
44
'checks membership in the set, and conditionally appends to the output array. ',
45
'This is a common pattern in data processing pipelines where uniqueness constraints must be maintained.\n\n',
46
'Edge cases to consider include empty arrays, duplicate transformations that produce the same key, ',
47
'and items where the transform function itself is expensive.\n\n',
48
'The time complexity is **O(n)** and the space complexity is **O(n)** in the worst case when all items are unique.\n',
49
], 20)
50
.build(),
51
},
52
53
'large-codeblock': {
54
description: 'Single large TypeScript code block',
55
chunks: new ScenarioBuilder()
56
.stream([
57
'Here is the refactored implementation:\n\n',
58
'```typescript\n',
59
'import { EventEmitter } from "events";\n\n',
60
'interface CacheEntry<T> {\n value: T;\n expiresAt: number;\n accessCount: number;\n}\n\n',
61
'export class LRUCache<K, V> {\n',
62
' private readonly _map = new Map<K, CacheEntry<V>>();\n',
63
' private readonly _emitter = new EventEmitter();\n\n',
64
' constructor(\n private readonly _maxSize: number,\n private readonly _ttlMs: number = 60_000,\n ) {}\n\n',
65
' get(key: K): V | undefined {\n const entry = this._map.get(key);\n if (!entry) { return undefined; }\n',
66
' if (Date.now() > entry.expiresAt) {\n this._map.delete(key);\n this._emitter.emit("evict", key);\n return undefined;\n }\n',
67
' entry.accessCount++;\n this._map.delete(key);\n this._map.set(key, entry);\n return entry.value;\n }\n\n',
68
' set(key: K, value: V): void {\n if (this._map.size >= this._maxSize) {\n',
69
' const oldest = this._map.keys().next().value;\n if (oldest !== undefined) {\n this._map.delete(oldest);\n this._emitter.emit("evict", oldest);\n }\n }\n',
70
' this._map.set(key, { value, expiresAt: Date.now() + this._ttlMs, accessCount: 0 });\n }\n\n',
71
' clear(): void { this._map.clear(); this._emitter.emit("clear"); }\n',
72
' get size(): number { return this._map.size; }\n',
73
' onEvict(listener: (key: K) => void): void { this._emitter.on("evict", listener); }\n}\n',
74
'```\n\n',
75
'The key changes:\n- Added TTL-based expiry with configurable timeout\n- LRU eviction uses Map insertion order\n- EventEmitter notifies on evictions for cache observability\n',
76
], 20)
77
.build(),
78
},
79
80
'many-small-chunks': {
81
description: '200 word-level chunks at 5ms',
82
chunks: (() => {
83
const words = ['Generating detailed analysis:\n\n'];
84
for (let i = 0; i < 200; i++) { words.push(`Word${i} `); }
85
words.push('\n\nAnalysis complete.\n');
86
const b = new ScenarioBuilder();
87
b.stream(words, 5);
88
return b.build();
89
})(),
90
},
91
92
'mixed-content': {
93
description: 'Markdown + code block + fix suggestion',
94
chunks: new ScenarioBuilder()
95
.stream([
96
'## Issue Found\n\n',
97
'The `DisposableStore` is not being disposed in the `deactivate` path, ',
98
'which can lead to memory leaks.\n\n',
99
'### Current Code\n\n',
100
'```typescript\nclass MyService {\n private store = new DisposableStore();\n // missing dispose!\n}\n```\n\n',
101
'### Suggested Fix\n\n',
102
'```typescript\nclass MyService extends Disposable {\n',
103
' private readonly store = this._register(new DisposableStore());\n\n',
104
' override dispose(): void {\n this.store.dispose();\n super.dispose();\n }\n}\n```\n\n',
105
'This ensures the store is cleaned up when the service is disposed via the workbench lifecycle.\n',
106
], 20)
107
.build(),
108
},
109
110
// -- Stress-test scenarios --------------------------------------------
111
112
'many-codeblocks': {
113
description: '10 code blocks, 60 lines each',
114
chunks: (() => {
115
const b = new ScenarioBuilder();
116
b.emit('Here are the implementations for each module:\n\n');
117
for (let i = 0; i < 10; i++) {
118
b.wait(10, `### Module ${i + 1}: \`handler${i}.ts\`\n\n`);
119
b.emit('```typescript\n');
120
const lines = [];
121
for (let j = 0; j < 15; j++) {
122
lines.push(`export function handle${i}_${j}(input: string): string {\n`);
123
lines.push(` const result = input.trim().split('').reverse().join('');\n`);
124
lines.push(` return \`[\${result}] processed by handler ${i}_${j}\`;\n`);
125
lines.push('}\n\n');
126
}
127
b.stream(lines, 5);
128
b.emit('```\n\n');
129
}
130
b.emit('All modules implement the same pattern with unique handler IDs.\n');
131
return b.build();
132
})(),
133
},
134
135
'long-prose': {
136
description: '15 sections, ~3000 words of prose',
137
chunks: (() => {
138
const sentences = [
139
'The architecture follows a layered dependency injection pattern where each service declares its dependencies through constructor parameters. ',
140
'This approach ensures that circular dependencies are detected at compile time rather than at runtime, which significantly reduces debugging overhead. ',
141
'When a service is instantiated, the instantiation service resolves all of its dependencies recursively, creating a directed acyclic graph of service instances. ',
142
'Each service is a singleton within its scope, meaning that multiple consumers of the same service interface receive the same instance. ',
143
'The workbench lifecycle manages the creation and disposal of these services through well-defined phases: creation, restoration, and eventual shutdown. ',
144
'During the restoration phase, services that persist state across sessions reload their data from storage, which may involve asynchronous operations. ',
145
'Contributors register their functionality through extension points, which are processed during the appropriate lifecycle phase. ',
146
'This contribution model allows features to be added without modifying the core workbench code, maintaining a clean separation of concerns. ',
147
];
148
const b = new ScenarioBuilder();
149
b.emit('# Detailed Architecture Analysis\n\n');
150
for (let para = 0; para < 15; para++) {
151
b.wait(15, `## Section ${para + 1}: ${['Overview', 'Design Patterns', 'Service Layer', 'Event System', 'State Management', 'Error Handling', 'Performance', 'Testing', 'Deployment', 'Monitoring', 'Security', 'Extensibility', 'Compatibility', 'Migration', 'Future Work'][para]}\n\n`);
152
const paraSentences = [];
153
for (let s = 0; s < 25; s++) { paraSentences.push(sentences[s % sentences.length]); }
154
b.stream(paraSentences, 8);
155
b.emit('\n\n');
156
}
157
return b.build();
158
})(),
159
},
160
161
'rich-markdown': {
162
description: '6 sections × 5 items, bold/links/code spans',
163
chunks: (() => {
164
const b = new ScenarioBuilder();
165
b.emit('# Comprehensive Code Review Report\n\n');
166
b.wait(15, '> **Summary**: Found 12 issues across 4 severity levels.\n\n');
167
for (let section = 0; section < 6; section++) {
168
b.wait(10, `## ${section + 1}. ${['Critical Issues', 'Performance Concerns', 'Code Style', 'Documentation Gaps', 'Test Coverage', 'Security Review'][section]}\n\n`);
169
for (let item = 0; item < 5; item++) {
170
b.stream([
171
`${item + 1}. **Issue ${section * 5 + item + 1}**: \`${['useState', 'useEffect', 'useMemo', 'useCallback', 'useRef'][item]}\` in \`src/components/Widget${item}.tsx\`\n`,
172
` - Severity: ${['[Critical]', '[Warning]', '[Info]', '[Suggestion]', '[Note]'][item]}\n`,
173
` - The current implementation uses *unnecessary re-renders* due to missing dependency arrays.\n`,
174
` - See [React docs](https://react.dev/reference) and the [\`useMemo\` guide](https://react.dev/reference/react/useMemo).\n`,
175
` - Fix: wrap in \`useCallback\` or extract to a ***separate memoized component***.\n\n`,
176
], 10);
177
}
178
b.emit('---\n\n');
179
}
180
b.emit('> *Report generated automatically. Please review all suggestions before applying.*\n');
181
return b.build();
182
})(),
183
},
184
185
'giant-codeblock': {
186
description: '40 classes in one fenced code block',
187
chunks: (() => {
188
const b = new ScenarioBuilder();
189
b.emit('Here is the complete implementation:\n\n```typescript\n');
190
b.stream([
191
'import { Disposable, DisposableStore } from "vs/base/common/lifecycle";\n',
192
'import { Emitter, Event } from "vs/base/common/event";\n',
193
'import { URI } from "vs/base/common/uri";\n\n',
194
], 10);
195
for (let i = 0; i < 40; i++) {
196
b.stream([
197
`export class Service${i} extends Disposable {\n`,
198
` private readonly _onDidChange = this._register(new Emitter<void>());\n`,
199
` readonly onDidChange: Event<void> = this._onDidChange.event;\n\n`,
200
` private _value: string = '';\n`,
201
` get value(): string { return this._value; }\n\n`,
202
` async update(uri: URI): Promise<void> {\n`,
203
` this._value = uri.toString();\n`,
204
` this._onDidChange.fire();\n`,
205
` }\n`,
206
'}\n\n',
207
], 5);
208
}
209
b.emit('```\n\nThis defines 40 service classes following the standard VS Code pattern.\n');
210
return b.build();
211
})(),
212
},
213
214
'rapid-stream': {
215
description: '1000 tokens at 2ms (streaming stress test)',
216
chunks: (() => {
217
const b = new ScenarioBuilder();
218
const words = [];
219
for (let i = 0; i < 1000; i++) { words.push(`w${i} `); }
220
// Very fast inter-chunk delay to stress the streaming pipeline
221
b.stream(words, 2);
222
return b.build();
223
})(),
224
},
225
226
'file-links': {
227
description: '32 file references with line links',
228
chunks: (() => {
229
const files = [
230
'src/vs/workbench/contrib/chat/browser/chatListRenderer.ts',
231
'src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts',
232
'src/vs/workbench/contrib/chat/browser/widget/input/chatInputPart.ts',
233
'src/vs/workbench/contrib/chat/common/chatPerf.ts',
234
'src/vs/base/common/lifecycle.ts',
235
'src/vs/base/common/event.ts',
236
'src/vs/platform/instantiation/common/instantiation.ts',
237
'src/vs/workbench/services/extensions/common/abstractExtensionService.ts',
238
'src/vs/workbench/api/common/extHostLanguageModels.ts',
239
'src/vs/workbench/contrib/chat/common/languageModels.ts',
240
'src/vs/editor/browser/widget/codeEditor/editor.ts',
241
'src/vs/workbench/browser/parts/editor/editorGroupView.ts',
242
];
243
const b = new ScenarioBuilder();
244
b.emit('I found references to the disposable pattern across the following files:\n\n');
245
for (let i = 0; i < files.length; i++) {
246
const line = Math.floor(Math.random() * 500) + 1;
247
b.stream([
248
`${i + 1}. [${files[i]}](${files[i]}#L${line}) -- `,
249
`Line ${line}: uses \`DisposableStore\` with ${Math.floor(Math.random() * 10) + 1} registrations\n`,
250
], 15);
251
}
252
b.wait(10, '\nAdditionally, the following files import from `vs/base/common/lifecycle`:\n\n');
253
for (let i = 0; i < 20; i++) {
254
const depth = ['base', 'platform', 'editor', 'workbench'][i % 4];
255
const area = ['common', 'browser', 'node', 'electron-browser'][i % 4];
256
const name = ['service', 'provider', 'contribution', 'handler', 'manager'][i % 5];
257
const file = `src/vs/${depth}/${area}/${name}${i}.ts`;
258
b.stream([
259
`- [${file}](${file}#L${i * 10 + 5})`,
260
` -- imports \`Disposable\`, \`DisposableStore\`\n`,
261
], 12);
262
}
263
b.emit('\nTotal: 32 files reference the disposable pattern.\n');
264
return b.build();
265
})(),
266
},
267
};
268
269
// -- Tool call scenarios ------------------------------------------------------
270
271
/** @type {Record<string, MultiTurnScenarioDef>} */
272
const TOOL_CALL_SCENARIOS = {
273
'tool-read-file': {
274
description: 'Read 8 files across 2 tool-call rounds',
275
scenario: /** @type {import('./mock-llm-server').MultiTurnScenario} */ ((() => {
276
const filesToRead = [
277
'_chatperf_lifecycle.ts',
278
'_chatperf_event.ts',
279
'_chatperf_uri.ts',
280
'_chatperf_errors.ts',
281
'_chatperf_async.ts',
282
'_chatperf_strings.ts',
283
'_chatperf_arrays.ts',
284
'_chatperf_types.ts',
285
];
286
// Round 1: parallel read of first 4 files
287
// Round 2: parallel read of next 4 files
288
// Round 3: final content response
289
return {
290
type: 'multi-turn',
291
turns: [
292
{
293
kind: 'tool-calls',
294
toolCalls: filesToRead.slice(0, 4).map(f => ({
295
toolNamePattern: /read.?file/i,
296
arguments: { filePath: path.join(FIXTURES_DIR, f), startLine: 1, endLine: 50 },
297
})),
298
},
299
{
300
kind: 'tool-calls',
301
toolCalls: filesToRead.slice(4).map(f => ({
302
toolNamePattern: /read.?file/i,
303
arguments: { filePath: path.join(FIXTURES_DIR, f), startLine: 1, endLine: 50 },
304
})),
305
},
306
{
307
kind: 'content',
308
chunks: new ScenarioBuilder()
309
.wait(20, '## Analysis of VS Code Base Utilities\n\n')
310
.stream([
311
'I read 8 core utility files from `src/vs/base/common/`. Here is a summary:\n\n',
312
'### lifecycle.ts\n',
313
'The `Disposable` base class provides the standard lifecycle pattern. Components register cleanup ',
314
'handlers via `this._register()` which are automatically disposed when the parent is disposed.\n\n',
315
'### event.ts\n',
316
'The `Emitter` class implements the observer pattern. `Event.once()`, `Event.map()`, and `Event.filter()` ',
317
'provide functional combinators for composing event streams.\n\n',
318
'### uri.ts\n',
319
'`URI` is an immutable representation of a resource identifier with scheme, authority, path, query, and fragment.\n\n',
320
'### errors.ts\n',
321
'Central error handling with `onUnexpectedError()` and `isCancellationError()` for distinguishing user cancellation.\n\n',
322
'### async.ts\n',
323
'`Throttler`, `Delayer`, `RunOnceScheduler`, and `Queue` manage async operation scheduling and deduplication.\n\n',
324
'### strings.ts\n',
325
'String utilities including `format()`, `escape()`, `startsWith()`, and `endsWith()` for common string operations.\n\n',
326
'### arrays.ts\n',
327
'Array helpers like `coalesce()`, `groupBy()`, `distinct()`, and binary search implementations.\n\n',
328
'### types.ts\n',
329
'Type guards and assertion helpers: `isString()`, `isNumber()`, `assertType()`, `assertIsDefined()`.\n',
330
], 15)
331
.build(),
332
},
333
],
334
};
335
})()),
336
},
337
338
'tool-edit-file': {
339
description: 'Read 3 files, edit 2 (read + write rounds)',
340
scenario: /** @type {import('./mock-llm-server').MultiTurnScenario} */ ((() => {
341
const readFiles = [
342
'_chatperf_lifecycle.ts',
343
'_chatperf_event.ts',
344
'_chatperf_errors.ts',
345
];
346
return {
347
type: 'multi-turn',
348
turns: [
349
// Round 1: read all 3 files in parallel
350
{
351
kind: 'tool-calls',
352
toolCalls: readFiles.map(f => ({
353
toolNamePattern: /read.?file/i,
354
arguments: { filePath: path.join(FIXTURES_DIR, f), startLine: 1, endLine: 40 },
355
})),
356
},
357
// Round 2: edit 2 files in parallel
358
{
359
kind: 'tool-calls',
360
toolCalls: [
361
{
362
toolNamePattern: /insert.?edit|replace.?string|apply.?patch/i,
363
arguments: {
364
filePath: path.join(FIXTURES_DIR, '_chatperf_lifecycle.ts'),
365
explanation: 'Update the benchmark marker comment in lifecycle.ts',
366
code: '// perf-benchmark-marker (updated)',
367
},
368
},
369
{
370
toolNamePattern: /insert.?edit|replace.?string|apply.?patch/i,
371
arguments: {
372
filePath: path.join(FIXTURES_DIR, '_chatperf_event.ts'),
373
explanation: 'Update the benchmark marker comment in event.ts',
374
code: '// perf-benchmark-marker (updated)',
375
},
376
},
377
],
378
},
379
// Round 3: final content
380
{
381
kind: 'content',
382
chunks: new ScenarioBuilder()
383
.wait(20, '## Edits Applied\n\n')
384
.stream([
385
'I read 3 files and applied edits to 2 of them:\n\n',
386
'### Files read:\n',
387
'1. `src/vs/base/common/lifecycle.ts` — Disposable pattern and lifecycle management\n',
388
'2. `src/vs/base/common/event.ts` — Event emitter and observer pattern\n',
389
'3. `src/vs/base/common/errors.ts` — Error handling utilities\n\n',
390
'### Edits applied:\n',
391
'1. **lifecycle.ts** — Updated the benchmark marker comment\n',
392
'2. **event.ts** — Updated the benchmark marker comment\n\n',
393
'Both files follow the standard VS Code pattern of using `Disposable` as a base class ',
394
'with `_register()` for lifecycle management. The edits were minimal and localized.\n',
395
], 20)
396
.build(),
397
},
398
],
399
};
400
})()),
401
},
402
403
'tool-terminal': {
404
description: 'Run commands, read output, fix + rerun',
405
scenario: /** @type {import('./mock-llm-server').MultiTurnScenario} */ ({
406
type: 'multi-turn',
407
turns: [
408
// Round 1: run initial commands (install + build)
409
{
410
kind: 'tool-calls',
411
toolCalls: [
412
{
413
toolNamePattern: /run.?in.?terminal|execute.?command/i,
414
arguments: {
415
command: 'echo "Installing dependencies..." && echo "added 1631 packages in 6m"',
416
explanation: 'Install project dependencies',
417
goal: 'Install dependencies',
418
mode: 'sync',
419
timeout: 30000,
420
},
421
},
422
],
423
},
424
// Round 2: run test command
425
{
426
kind: 'tool-calls',
427
toolCalls: [
428
{
429
toolNamePattern: /run.?in.?terminal|execute.?command/i,
430
arguments: {
431
command: 'echo "Running unit tests..." && echo " 42 passing (3s)" && echo " 2 failing" && echo "" && echo " 1) ChatService should dispose listeners" && echo " AssertionError: expected 0 to equal 1" && echo " 2) ChatModel should clear on new session" && echo " TypeError: Cannot read property dispose of undefined"',
432
explanation: 'Run the unit test suite to check for failures',
433
goal: 'Run tests',
434
mode: 'sync',
435
timeout: 60000,
436
},
437
},
438
],
439
},
440
// Round 3: read the failing test file for context
441
{
442
kind: 'tool-calls',
443
toolCalls: [
444
{
445
toolNamePattern: /read.?file/i,
446
arguments: { filePath: path.join(FIXTURES_DIR, '_chatperf_lifecycle.ts'), startLine: 1, endLine: 50 },
447
},
448
],
449
},
450
// Round 4: fix the issue with an edit
451
{
452
kind: 'tool-calls',
453
toolCalls: [
454
{
455
toolNamePattern: /insert.?edit|replace.?string|apply.?patch/i,
456
arguments: {
457
filePath: path.join(FIXTURES_DIR, '_chatperf_lifecycle.ts'),
458
explanation: 'Fix the dispose call in the test',
459
code: '// perf-benchmark-marker (fixed)',
460
},
461
},
462
],
463
},
464
// Round 5: re-run tests to confirm
465
{
466
kind: 'tool-calls',
467
toolCalls: [
468
{
469
toolNamePattern: /run.?in.?terminal|execute.?command/i,
470
arguments: {
471
command: 'echo "Running unit tests..." && echo " 44 passing (3s)" && echo " 0 failing"',
472
explanation: 'Re-run tests to verify the fix',
473
goal: 'Verify fix',
474
mode: 'sync',
475
timeout: 60000,
476
},
477
},
478
],
479
},
480
// Round 6: final summary
481
{
482
kind: 'content',
483
chunks: new ScenarioBuilder()
484
.wait(20, '## Test Failures Fixed\n\n')
485
.stream([
486
'I found and fixed 2 test failures:\n\n',
487
'### Root Cause\n',
488
'The `ChatService` was not properly disposing event listeners when a session was cleared. ',
489
'The `dispose()` method was missing a call to `this._store.dispose()`.\n\n',
490
'### Changes Made\n',
491
'Updated `lifecycle.ts` to properly chain disposal:\n\n',
492
'```typescript\n',
493
'override dispose(): void {\n',
494
' this._store.dispose();\n',
495
' super.dispose();\n',
496
'}\n',
497
'```\n\n',
498
'### Test Results\n',
499
'- **Before**: 42 passing, 2 failing\n',
500
'- **After**: 44 passing, 0 failing\n\n',
501
'All tests pass now. The fix ensures listeners are cleaned up during session transitions.\n',
502
], 15)
503
.build(),
504
},
505
],
506
}),
507
},
508
};
509
510
// -- Multi-turn user conversation scenarios -----------------------------------
511
512
/** @type {Record<string, MultiTurnScenarioDef>} */
513
const MULTI_TURN_SCENARIOS = {
514
'thinking-response': {
515
description: 'Thinking block before content response',
516
scenario: /** @type {import('./mock-llm-server').MultiTurnScenario} */ ({
517
type: 'multi-turn',
518
turns: [
519
{
520
kind: 'thinking',
521
thinkingChunks: new ScenarioBuilder()
522
.stream([
523
'Let me analyze this code carefully. ',
524
'The user is asking about the lifecycle pattern in VS Code. ',
525
'I should look at the Disposable base class and how it manages cleanup. ',
526
'The key methods are _register(), dispose(), and the DisposableStore pattern. ',
527
'I need to read the file first to give an accurate explanation.',
528
], 15)
529
.build(),
530
chunks: new ScenarioBuilder()
531
.wait(20, 'I\'ll start by reading the file to understand its structure.\n\n')
532
.stream([
533
'The `Disposable` base class in `lifecycle.ts` provides a standard pattern ',
534
'for managing resources. It uses a `DisposableStore` internally to track ',
535
'all registered disposables and clean them up on `dispose()`.\n',
536
], 20)
537
.build(),
538
},
539
],
540
}),
541
},
542
543
'multi-turn-user': {
544
description: '2 user follow-ups with thinking + code',
545
scenario: /** @type {import('./mock-llm-server').MultiTurnScenario} */ ({
546
type: 'multi-turn',
547
turns: [
548
// Turn 1: Model reads a file
549
{
550
kind: 'tool-calls',
551
toolCalls: [
552
{
553
toolNamePattern: /read.?file/i,
554
arguments: {
555
filePath: path.join(FIXTURES_DIR, '_chatperf_lifecycle.ts'),
556
offset: 1,
557
limit: 50,
558
},
559
},
560
],
561
},
562
// Turn 2: Model responds with analysis
563
{
564
kind: 'content',
565
chunks: new ScenarioBuilder()
566
.wait(20, 'I\'ve read the file. Here\'s what I found:\n\n')
567
.stream([
568
'The `Disposable` class is the base for lifecycle management. ',
569
'It internally holds a `DisposableStore` via `this._store`. ',
570
'Subclasses call `this._register()` to track their own disposables.\n\n',
571
'Would you like me to explain any specific part in more detail?\n',
572
], 20)
573
.build(),
574
},
575
// Turn 3: User follow-up (injected by test harness, not served by mock)
576
{
577
kind: 'user',
578
message: 'Yes, explain the MutableDisposable pattern',
579
},
580
// Turn 4: Model responds with thinking, then content
581
{
582
kind: 'thinking',
583
thinkingChunks: new ScenarioBuilder()
584
.stream([
585
'The user wants to understand MutableDisposable specifically. ',
586
'Let me recall the key aspects: it holds a single disposable that can be swapped. ',
587
'When a new value is set, the old one is automatically disposed. ',
588
'This is useful for things like event listener subscriptions that need to be replaced.',
589
], 10)
590
.build(),
591
chunks: new ScenarioBuilder()
592
.wait(15, '## MutableDisposable\n\n')
593
.stream([
594
'`MutableDisposable<T>` holds a **single disposable** that can be swapped at any time. ',
595
'When you set a new value via `.value = newDisposable`, the previous value is automatically disposed.\n\n',
596
'This is perfect for:\n',
597
'- **Event listeners** that need to be re-subscribed when configuration changes\n',
598
'- **Editor decorations** that are replaced when content updates\n',
599
'- **Watchers** that switch targets dynamically\n\n',
600
'```typescript\n',
601
'class MyService extends Disposable {\n',
602
' private readonly _listener = this._register(new MutableDisposable());\n\n',
603
' updateTarget(editor: ICodeEditor): void {\n',
604
' // Old listener is automatically disposed\n',
605
' this._listener.value = editor.onDidChangeModel(() => {\n',
606
' this._handleModelChange();\n',
607
' });\n',
608
' }\n',
609
'}\n',
610
'```\n\n',
611
'The key benefit is that you never forget to dispose the old subscription.\n',
612
], 15)
613
.build(),
614
},
615
// Turn 5: Second user follow-up
616
{
617
kind: 'user',
618
message: 'Can you also show me DisposableMap?',
619
},
620
// Turn 6: Final response
621
{
622
kind: 'content',
623
chunks: new ScenarioBuilder()
624
.wait(20, '## DisposableMap\n\n')
625
.stream([
626
'`DisposableMap<K, V>` extends `Map` with automatic disposal semantics:\n\n',
627
'- When a key is **overwritten**, the old value is disposed\n',
628
'- When a key is **deleted**, the value is disposed\n',
629
'- When the map itself is **disposed**, all values are disposed\n\n',
630
'```typescript\n',
631
'class ToolManager extends Disposable {\n',
632
' private readonly _tools = this._register(new DisposableMap<string, IDisposable>());\n\n',
633
' registerTool(id: string, tool: IDisposable): void {\n',
634
' this._tools.set(id, tool); // auto-disposes previous tool with same id\n',
635
' }\n',
636
'}\n',
637
'```\n\n',
638
'This is commonly used for managing collections of disposable resources keyed by ID.\n',
639
], 15)
640
.build(),
641
},
642
],
643
}),
644
},
645
'long-conversation': {
646
description: '10 user turns, mixed content types',
647
scenario: /** @type {import('./mock-llm-server').MultiTurnScenario} */ ((() => {
648
const topics = [
649
{ question: 'How does the Disposable pattern work?', heading: 'Disposable Pattern', content: 'The `Disposable` base class provides lifecycle management. Subclasses call `this._register()` to track child disposables that are cleaned up automatically when `dispose()` is called.' },
650
{ question: 'What about DisposableStore?', heading: 'DisposableStore', content: '`DisposableStore` aggregates multiple `IDisposable` instances and disposes them all at once. It tracks whether it has already been disposed and throws if you try to add after disposal.' },
651
{ question: 'How does the Event system work?', heading: 'Event System', content: 'The `Emitter<T>` class implements the observer pattern. `Event.once()`, `Event.map()`, `Event.filter()`, and `Event.debounce()` provide functional combinators for composing event streams.' },
652
{ question: 'Explain dependency injection', heading: 'Dependency Injection', content: 'Services are injected through constructor parameters decorated with service identifiers. The `IInstantiationService` resolves dependencies recursively, creating singletons within each scope.' },
653
{ question: 'What is the contribution model?', heading: 'Contribution Model', content: 'Features register functionality through extension points like `Registry.as<IWorkbenchContributionsRegistry>()`. Contributions are instantiated during specific lifecycle phases.' },
654
{ question: 'How does the editor handle text models?', heading: 'Text Models', content: 'The `TextModel` class manages document content with line-based storage. It supports undo/redo stacks, bracket matching, tokenization, and change tracking via edit operations.' },
655
{ question: 'Explain the extension host architecture', heading: 'Extension Host', content: 'Extensions run in a separate process (or worker) called the extension host. Communication happens via an RPC protocol over `IPC`. The main process proxies API calls back to the workbench.' },
656
{ question: 'How does file watching work?', heading: 'File Watching', content: 'The `IFileService` supports correlated and shared file watchers. Correlated watchers are preferred as they track specific resources. The underlying implementation uses `chokidar` or `parcel/watcher`.' },
657
{ question: 'What about the tree widget?', heading: 'Tree Widget', content: 'The `AsyncDataTree` and `ObjectTree` provide virtualized tree rendering. They support filtering, sorting, keyboard navigation, and accessibility. The `ITreeRenderer` interface handles element rendering.' },
658
{ question: 'How does the settings editor work?', heading: 'Settings Editor', content: 'Settings are declared in `package.json` contribution points. The settings editor reads the configuration registry, groups settings by category, and renders appropriate input controls for each type.' },
659
];
660
661
/** @type {import('./mock-llm-server').ScenarioTurn[]} */
662
const turns = [];
663
664
// Turn 1: Initial model response (no user turn needed before the first)
665
const firstTopic = topics[0];
666
turns.push({
667
kind: 'content',
668
chunks: new ScenarioBuilder()
669
.wait(20, `## ${firstTopic.heading}\n\n`)
670
.stream([
671
`${firstTopic.content}\n\n`,
672
'Here is a typical example:\n\n',
673
'```typescript\n',
674
'class MyService extends Disposable {\n',
675
' private readonly _onDidChange = this._register(new Emitter<void>());\n',
676
' readonly onDidChange: Event<void> = this._onDidChange.event;\n\n',
677
' constructor(@IFileService private readonly fileService: IFileService) {\n',
678
' super();\n',
679
' this._register(fileService.onDidFilesChange(e => this._handleChange(e)));\n',
680
' }\n',
681
'}\n',
682
'```\n\n',
683
'Would you like to know more about any specific aspect?\n',
684
], 15)
685
.build(),
686
});
687
688
// Turns 2..N: alternating user follow-up + model response
689
for (let i = 1; i < topics.length; i++) {
690
const topic = topics[i];
691
692
// User follow-up
693
turns.push({ kind: 'user', message: topic.question });
694
695
// Model response — vary content type to stress different renderers
696
const b = new ScenarioBuilder();
697
b.wait(20, `## ${topic.heading}\n\n`);
698
699
// Main explanation
700
const sentences = topic.content.split('. ');
701
b.stream(sentences.map(s => s.endsWith('.') ? s + ' ' : s + '. '), 12);
702
b.emit('\n\n');
703
704
if (i % 3 === 0) {
705
// Every 3rd response: large code block
706
b.emit('```typescript\n');
707
for (let j = 0; j < 8; j++) {
708
b.stream([
709
`export class ${topic.heading.replace(/\s/g, '')}Part${j} extends Disposable {\n`,
710
` private readonly _state = new Map<string, unknown>();\n\n`,
711
` process(input: string): string {\n`,
712
` const cached = this._state.get(input);\n`,
713
` if (cached) { return String(cached); }\n`,
714
` const result = input.split('').reverse().join('');\n`,
715
` this._state.set(input, result);\n`,
716
` return result;\n`,
717
` }\n`,
718
'}\n\n',
719
], 5);
720
}
721
b.emit('```\n\n');
722
} else if (i % 3 === 1) {
723
// Every 3rd+1 response: bullet list with bold + inline code
724
b.emit('Key points to remember:\n\n');
725
for (let j = 0; j < 6; j++) {
726
b.stream([
727
`${j + 1}. **Point ${j + 1}**: The \`${topic.heading.replace(/\s/g, '')}${j}\` `,
728
`component uses the standard pattern with \`_register()\` for lifecycle. `,
729
`It handles edge cases like ${['empty input', 'null references', 'concurrent access', 'circular deps', 'timeout expiry', 'disposal races'][j]}.\n`,
730
], 10);
731
}
732
b.emit('\n');
733
} else {
734
// Every 3rd+2 response: mixed prose + small code snippet
735
b.stream([
736
'This pattern is used extensively throughout the codebase. ',
737
'The key insight is that resources are always tracked from creation, ',
738
'ensuring no leaks even in error paths. ',
739
'The ownership chain is explicit and follows the component hierarchy.\n\n',
740
], 12);
741
b.emit('Quick example:\n\n```typescript\n');
742
b.stream([
743
`const store = new DisposableStore();\n`,
744
`store.add(event.on(() => { /* handler */ }));\n`,
745
`store.add(watcher.watch(uri));\n`,
746
`// Later: store.dispose(); // cleans up everything\n`,
747
], 8);
748
b.emit('```\n\n');
749
}
750
751
b.stream([
752
`That covers the essentials of **${topic.heading}**. `,
753
'Let me know if you want to dive deeper into any of these concepts.\n',
754
], 15);
755
756
turns.push({
757
kind: 'content',
758
chunks: b.build(),
759
});
760
}
761
762
return { type: 'multi-turn', turns };
763
})()),
764
},
765
};
766
767
// -- Registration helper ------------------------------------------------------
768
769
/**
770
* Get a brief description of a scenario by ID.
771
* @param {string} id
772
* @returns {string}
773
*/
774
function getScenarioDescription(id) {
775
const content = CONTENT_SCENARIOS[id];
776
if (content) { return content.description; }
777
const tool = TOOL_CALL_SCENARIOS[id];
778
if (tool) { return tool.description; }
779
const multi = MULTI_TURN_SCENARIOS[id];
780
if (multi) { return multi.description; }
781
return '';
782
}
783
784
/**
785
* Register all built-in perf scenarios into the mock LLM server.
786
* Call this from your test file before starting the server.
787
*/
788
function registerPerfScenarios() {
789
for (const [id, def] of Object.entries(CONTENT_SCENARIOS)) {
790
registerScenario(id, def.chunks);
791
}
792
for (const [id, def] of Object.entries(TOOL_CALL_SCENARIOS)) {
793
registerScenario(id, def.scenario);
794
}
795
for (const [id, def] of Object.entries(MULTI_TURN_SCENARIOS)) {
796
registerScenario(id, def.scenario);
797
}
798
}
799
800
module.exports = { registerPerfScenarios, getScenarioDescription, CONTENT_SCENARIOS, TOOL_CALL_SCENARIOS, MULTI_TURN_SCENARIOS };
801
802