CoCalc -- tokensStore.test.ts

GitHub Repository: microsoft/vscode
Path: blob/main/src/vs/editor/test/common/model/tokensStore.test.ts
³²⁹⁶ views
1
/*---------------------------------------------------------------------------------------------
2
 *  Copyright (c) Microsoft Corporation. All rights reserved.
3
 *  Licensed under the MIT License. See License.txt in the project root for license information.
4
 *--------------------------------------------------------------------------------------------*/
5

6
import assert from 'assert';
7
import { DisposableStore } from '../../../../base/common/lifecycle.js';
8
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js';
9
import { ISingleEditOperation } from '../../../common/core/editOperation.js';
10
import { Position } from '../../../common/core/position.js';
11
import { Range } from '../../../common/core/range.js';
12
import { ColorId, FontStyle, MetadataConsts, TokenMetadata } from '../../../common/encodedTokenAttributes.js';
13
import { ILanguageConfigurationService, LanguageConfigurationService } from '../../../common/languages/languageConfigurationRegistry.js';
14
import { TextModel } from '../../../common/model/textModel.js';
15
import { LanguageIdCodec } from '../../../common/services/languagesRegistry.js';
16
import { LineTokens } from '../../../common/tokens/lineTokens.js';
17
import { SparseMultilineTokens } from '../../../common/tokens/sparseMultilineTokens.js';
18
import { SparseTokensStore } from '../../../common/tokens/sparseTokensStore.js';
19
import { createModelServices, createTextModel, instantiateTextModel } from '../testTextModel.js';
20

21
suite('TokensStore', () => {
22

23
	ensureNoDisposablesAreLeakedInTestSuite();
24

25
	const SEMANTIC_COLOR = 5 as ColorId;
26

27
	function parseTokensState(state: string[]): { text: string; tokens: SparseMultilineTokens } {
28
		const text: string[] = [];
29
		const tokens: number[] = [];
30
		let baseLine = 1;
31
		for (let i = 0; i < state.length; i++) {
32
			const line = state[i];
33

34
			let startOffset = 0;
35
			let lineText = '';
36
			while (true) {
37
				const firstPipeOffset = line.indexOf('|', startOffset);
38
				if (firstPipeOffset === -1) {
39
					break;
40
				}
41
				const secondPipeOffset = line.indexOf('|', firstPipeOffset + 1);
42
				if (secondPipeOffset === -1) {
43
					break;
44
				}
45
				if (firstPipeOffset + 1 === secondPipeOffset) {
46
					// skip ||
47
					lineText += line.substring(startOffset, secondPipeOffset + 1);
48
					startOffset = secondPipeOffset + 1;
49
					continue;
50
				}
51

52
				lineText += line.substring(startOffset, firstPipeOffset);
53
				const tokenStartCharacter = lineText.length;
54
				const tokenLength = secondPipeOffset - firstPipeOffset - 1;
55
				const metadata = (
56
					SEMANTIC_COLOR << MetadataConsts.FOREGROUND_OFFSET
57
					| MetadataConsts.SEMANTIC_USE_FOREGROUND
58
				);
59

60
				if (tokens.length === 0) {
61
					baseLine = i + 1;
62
				}
63
				tokens.push(i + 1 - baseLine, tokenStartCharacter, tokenStartCharacter + tokenLength, metadata);
64

65
				lineText += line.substr(firstPipeOffset + 1, tokenLength);
66
				startOffset = secondPipeOffset + 1;
67
			}
68

69
			lineText += line.substring(startOffset);
70

71
			text.push(lineText);
72
		}
73

74
		return {
75
			text: text.join('\n'),
76
			tokens: SparseMultilineTokens.create(baseLine, new Uint32Array(tokens))
77
		};
78
	}
79

80
	function extractState(model: TextModel): string[] {
81
		const result: string[] = [];
82
		for (let lineNumber = 1; lineNumber <= model.getLineCount(); lineNumber++) {
83
			const lineTokens = model.tokenization.getLineTokens(lineNumber);
84
			const lineContent = model.getLineContent(lineNumber);
85

86
			let lineText = '';
87
			for (let i = 0; i < lineTokens.getCount(); i++) {
88
				const tokenStartCharacter = lineTokens.getStartOffset(i);
89
				const tokenEndCharacter = lineTokens.getEndOffset(i);
90
				const metadata = lineTokens.getMetadata(i);
91
				const color = TokenMetadata.getForeground(metadata);
92
				const tokenText = lineContent.substring(tokenStartCharacter, tokenEndCharacter);
93
				if (color === SEMANTIC_COLOR) {
94
					lineText += `|${tokenText}|`;
95
				} else {
96
					lineText += tokenText;
97
				}
98
			}
99

100
			result.push(lineText);
101
		}
102
		return result;
103
	}
104

105
	function testTokensAdjustment(rawInitialState: string[], edits: ISingleEditOperation[], rawFinalState: string[]) {
106
		const initialState = parseTokensState(rawInitialState);
107
		const model = createTextModel(initialState.text);
108
		model.tokenization.setSemanticTokens([initialState.tokens], true);
109

110
		model.applyEdits(edits);
111

112
		const actualState = extractState(model);
113
		assert.deepStrictEqual(actualState, rawFinalState);
114

115
		model.dispose();
116
	}
117

118
	test('issue #86303 - color shifting between different tokens', () => {
119
		testTokensAdjustment(
120
			[
121
				`import { |URI| } from 'vs/base/common/uri';`,
122
				`const foo = |URI|.parse('hey');`
123
			],
124
			[
125
				{ range: new Range(2, 9, 2, 10), text: '' }
126
			],
127
			[
128
				`import { |URI| } from 'vs/base/common/uri';`,
129
				`const fo = |URI|.parse('hey');`
130
			]
131
		);
132
	});
133

134
	test('deleting a newline', () => {
135
		testTokensAdjustment(
136
			[
137
				`import { |URI| } from 'vs/base/common/uri';`,
138
				`const foo = |URI|.parse('hey');`
139
			],
140
			[
141
				{ range: new Range(1, 42, 2, 1), text: '' }
142
			],
143
			[
144
				`import { |URI| } from 'vs/base/common/uri';const foo = |URI|.parse('hey');`
145
			]
146
		);
147
	});
148

149
	test('inserting a newline', () => {
150
		testTokensAdjustment(
151
			[
152
				`import { |URI| } from 'vs/base/common/uri';const foo = |URI|.parse('hey');`
153
			],
154
			[
155
				{ range: new Range(1, 42, 1, 42), text: '\n' }
156
			],
157
			[
158
				`import { |URI| } from 'vs/base/common/uri';`,
159
				`const foo = |URI|.parse('hey');`
160
			]
161
		);
162
	});
163

164
	test('deleting a newline 2', () => {
165
		testTokensAdjustment(
166
			[
167
				`import { `,
168
				`    |URI| } from 'vs/base/common/uri';const foo = |URI|.parse('hey');`
169
			],
170
			[
171
				{ range: new Range(1, 10, 2, 5), text: '' }
172
			],
173
			[
174
				`import { |URI| } from 'vs/base/common/uri';const foo = |URI|.parse('hey');`
175
			]
176
		);
177
	});
178

179
	test('issue #179268: a complex edit', () => {
180
		testTokensAdjustment(
181
			[
182
				`|export| |'interior_material_selector.dart'|;`,
183
				`|export| |'mileage_selector.dart'|;`,
184
				`|export| |'owners_selector.dart'|;`,
185
				`|export| |'price_selector.dart'|;`,
186
				`|export| |'seat_count_selector.dart'|;`,
187
				`|export| |'year_selector.dart'|;`,
188
				`|export| |'winter_options_selector.dart'|;|export| |'camera_selector.dart'|;`
189
			],
190
			[
191
				{ range: new Range(1, 9, 1, 9), text: `camera_selector.dart';\nexport '` },
192
				{ range: new Range(6, 9, 7, 9), text: `` },
193
				{ range: new Range(7, 39, 7, 39), text: `\n` },
194
				{ range: new Range(7, 47, 7, 48), text: `ye` },
195
				{ range: new Range(7, 49, 7, 51), text: `` },
196
				{ range: new Range(7, 52, 7, 53), text: `` },
197
			],
198
			[
199
				`|export| |'|camera_selector.dart';`,
200
				`export 'interior_material_selector.dart';`,
201
				`|export| |'mileage_selector.dart'|;`,
202
				`|export| |'owners_selector.dart'|;`,
203
				`|export| |'price_selector.dart'|;`,
204
				`|export| |'seat_count_selector.dart'|;`,
205
				`|export| |'||winter_options_selector.dart'|;`,
206
				`|export| |'year_selector.dart'|;`
207
			]
208
		);
209
	});
210

211
	test('issue #91936: Semantic token color highlighting fails on line with selected text', () => {
212
		const model = createTextModel('                    else if ($s = 08) then \'\\b\'');
213
		model.tokenization.setSemanticTokens([
214
			SparseMultilineTokens.create(1, new Uint32Array([
215
				0, 20, 24, 0b01111000000000010000,
216
				0, 25, 27, 0b01111000000000010000,
217
				0, 28, 29, 0b00001000000000010000,
218
				0, 29, 31, 0b10000000000000010000,
219
				0, 32, 33, 0b00001000000000010000,
220
				0, 34, 36, 0b00110000000000010000,
221
				0, 36, 37, 0b00001000000000010000,
222
				0, 38, 42, 0b01111000000000010000,
223
				0, 43, 47, 0b01011000000000010000,
224
			]))
225
		], true);
226
		const lineTokens = model.tokenization.getLineTokens(1);
227
		const decodedTokens: number[] = [];
228
		for (let i = 0, len = lineTokens.getCount(); i < len; i++) {
229
			decodedTokens.push(lineTokens.getEndOffset(i), lineTokens.getMetadata(i));
230
		}
231

232
		assert.deepStrictEqual(decodedTokens, [
233
			20, 0b10000000001000010000000001,
234
			24, 0b10000001111000010000000001,
235
			25, 0b10000000001000010000000001,
236
			27, 0b10000001111000010000000001,
237
			28, 0b10000000001000010000000001,
238
			29, 0b10000000001000010000000001,
239
			31, 0b10000010000000010000000001,
240
			32, 0b10000000001000010000000001,
241
			33, 0b10000000001000010000000001,
242
			34, 0b10000000001000010000000001,
243
			36, 0b10000000110000010000000001,
244
			37, 0b10000000001000010000000001,
245
			38, 0b10000000001000010000000001,
246
			42, 0b10000001111000010000000001,
247
			43, 0b10000000001000010000000001,
248
			47, 0b10000001011000010000000001
249
		]);
250

251
		model.dispose();
252
	});
253

254
	test('issue #147944: Language id "vs.editor.nullLanguage" is not configured nor known', () => {
255
		const disposables = new DisposableStore();
256
		const instantiationService = createModelServices(disposables, [
257
			[ILanguageConfigurationService, LanguageConfigurationService]
258
		]);
259
		const model = disposables.add(instantiateTextModel(instantiationService, '--[[\n\n]]'));
260
		model.tokenization.setSemanticTokens([
261
			SparseMultilineTokens.create(1, new Uint32Array([
262
				0, 2, 4, 0b100000000000010000,
263
				1, 0, 0, 0b100000000000010000,
264
				2, 0, 2, 0b100000000000010000,
265
			]))
266
		], true);
267
		assert.strictEqual(model.getWordAtPosition(new Position(2, 1)), null);
268
		disposables.dispose();
269
	});
270

271
	test('partial tokens 1', () => {
272
		const codec = new LanguageIdCodec();
273
		const store = new SparseTokensStore(codec);
274

275
		// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
276
		store.setPartial(new Range(1, 1, 31, 2), [
277
			SparseMultilineTokens.create(5, new Uint32Array([
278
				0, 5, 10, 1,
279
				5, 5, 10, 2,
280
				10, 5, 10, 3,
281
				15, 5, 10, 4,
282
				20, 5, 10, 5,
283
				25, 5, 10, 6,
284
			]))
285
		]);
286

287
		// setPartial: [18,1 -> 42,1], [(20,5-10),(25,5-10),(30,5-10),(35,5-10),(40,5-10)]
288
		store.setPartial(new Range(18, 1, 42, 1), [
289
			SparseMultilineTokens.create(20, new Uint32Array([
290
				0, 5, 10, 4,
291
				5, 5, 10, 5,
292
				10, 5, 10, 6,
293
				15, 5, 10, 7,
294
				20, 5, 10, 8,
295
			]))
296
		]);
297

298
		// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
299
		store.setPartial(new Range(1, 1, 31, 2), [
300
			SparseMultilineTokens.create(5, new Uint32Array([
301
				0, 5, 10, 1,
302
				5, 5, 10, 2,
303
				10, 5, 10, 3,
304
				15, 5, 10, 4,
305
				20, 5, 10, 5,
306
				25, 5, 10, 6,
307
			]))
308
		]);
309

310
		const lineTokens = store.addSparseTokens(10, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
311
		assert.strictEqual(lineTokens.getCount(), 3);
312
	});
313

314
	test('partial tokens 2', () => {
315
		const codec = new LanguageIdCodec();
316
		const store = new SparseTokensStore(codec);
317

318
		// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
319
		store.setPartial(new Range(1, 1, 31, 2), [
320
			SparseMultilineTokens.create(5, new Uint32Array([
321
				0, 5, 10, 1,
322
				5, 5, 10, 2,
323
				10, 5, 10, 3,
324
				15, 5, 10, 4,
325
				20, 5, 10, 5,
326
				25, 5, 10, 6,
327
			]))
328
		]);
329

330
		// setPartial: [6,1 -> 36,2], [(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10),(35,5-10)]
331
		store.setPartial(new Range(6, 1, 36, 2), [
332
			SparseMultilineTokens.create(10, new Uint32Array([
333
				0, 5, 10, 2,
334
				5, 5, 10, 3,
335
				10, 5, 10, 4,
336
				15, 5, 10, 5,
337
				20, 5, 10, 6,
338
			]))
339
		]);
340

341
		// setPartial: [17,1 -> 42,1], [(20,5-10),(25,5-10),(30,5-10),(35,5-10),(40,5-10)]
342
		store.setPartial(new Range(17, 1, 42, 1), [
343
			SparseMultilineTokens.create(20, new Uint32Array([
344
				0, 5, 10, 4,
345
				5, 5, 10, 5,
346
				10, 5, 10, 6,
347
				15, 5, 10, 7,
348
				20, 5, 10, 8,
349
			]))
350
		]);
351

352
		const lineTokens = store.addSparseTokens(20, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
353
		assert.strictEqual(lineTokens.getCount(), 3);
354
	});
355

356
	test('partial tokens 3', () => {
357
		const codec = new LanguageIdCodec();
358
		const store = new SparseTokensStore(codec);
359

360
		// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
361
		store.setPartial(new Range(1, 1, 31, 2), [
362
			SparseMultilineTokens.create(5, new Uint32Array([
363
				0, 5, 10, 1,
364
				5, 5, 10, 2,
365
				10, 5, 10, 3,
366
				15, 5, 10, 4,
367
				20, 5, 10, 5,
368
				25, 5, 10, 6,
369
			]))
370
		]);
371

372
		// setPartial: [11,1 -> 16,2], [(15,5-10),(20,5-10)]
373
		store.setPartial(new Range(11, 1, 16, 2), [
374
			SparseMultilineTokens.create(10, new Uint32Array([
375
				0, 5, 10, 3,
376
				5, 5, 10, 4,
377
			]))
378
		]);
379

380
		const lineTokens = store.addSparseTokens(5, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
381
		assert.strictEqual(lineTokens.getCount(), 3);
382
	});
383

384
	test('issue #94133: Semantic colors stick around when using (only) range provider', () => {
385
		const codec = new LanguageIdCodec();
386
		const store = new SparseTokensStore(codec);
387

388
		// setPartial: [1,1 -> 1,20] [(1,9-11)]
389
		store.setPartial(new Range(1, 1, 1, 20), [
390
			SparseMultilineTokens.create(1, new Uint32Array([
391
				0, 9, 11, 1,
392
			]))
393
		]);
394

395
		// setPartial: [1,1 -> 1,20], []
396
		store.setPartial(new Range(1, 1, 1, 20), []);
397

398
		const lineTokens = store.addSparseTokens(1, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
399
		assert.strictEqual(lineTokens.getCount(), 1);
400
	});
401

402
	test('bug', () => {
403
		function createTokens(str: string): SparseMultilineTokens {
404
			str = str.replace(/^\[\(/, '');
405
			str = str.replace(/\)\]$/, '');
406
			const strTokens = str.split('),(');
407
			const result: number[] = [];
408
			let firstLineNumber = 0;
409
			for (const strToken of strTokens) {
410
				const pieces = strToken.split(',');
411
				const chars = pieces[1].split('-');
412
				const lineNumber = parseInt(pieces[0], 10);
413
				const startChar = parseInt(chars[0], 10);
414
				const endChar = parseInt(chars[1], 10);
415
				if (firstLineNumber === 0) {
416
					// this is the first line
417
					firstLineNumber = lineNumber;
418
				}
419
				result.push(lineNumber - firstLineNumber, startChar, endChar, (lineNumber + startChar) % 13);
420
			}
421
			return SparseMultilineTokens.create(firstLineNumber, new Uint32Array(result));
422
		}
423

424
		const codec = new LanguageIdCodec();
425
		const store = new SparseTokensStore(codec);
426
		// setPartial [36446,1 -> 36475,115] [(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35),(36470,38-46),(36473,25-35),(36473,36-51),(36474,28-33),(36474,36-49),(36474,50-58),(36475,35-53),(36475,54-62)]
427
		store.setPartial(
428
			new Range(36446, 1, 36475, 115),
429
			[createTokens('[(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35),(36470,38-46),(36473,25-35),(36473,36-51),(36474,28-33),(36474,36-49),(36474,50-58),(36475,35-53),(36475,54-62)]')]
430
		);
431
		// setPartial [36436,1 -> 36464,142] [(36437,33-37),(36437,38-42),(36437,47-57),(36437,58-67),(36438,35-53),(36438,54-62),(36440,24-29),(36440,33-46),(36440,47-53),(36442,25-35),(36442,36-50),(36443,30-39),(36443,42-46),(36443,47-53),(36443,54-58),(36443,63-73),(36443,74-84),(36443,87-91),(36443,92-98),(36443,101-105),(36443,106-112),(36443,113-119),(36444,28-37),(36444,38-42),(36444,47-57),(36444,58-75),(36444,80-95),(36444,96-105),(36445,35-53),(36445,54-62),(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62)]
432
		store.setPartial(
433
			new Range(36436, 1, 36464, 142),
434
			[createTokens('[(36437,33-37),(36437,38-42),(36437,47-57),(36437,58-67),(36438,35-53),(36438,54-62),(36440,24-29),(36440,33-46),(36440,47-53),(36442,25-35),(36442,36-50),(36443,30-39),(36443,42-46),(36443,47-53),(36443,54-58),(36443,63-73),(36443,74-84),(36443,87-91),(36443,92-98),(36443,101-105),(36443,106-112),(36443,113-119),(36444,28-37),(36444,38-42),(36444,47-57),(36444,58-75),(36444,80-95),(36444,96-105),(36445,35-53),(36445,54-62),(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62)]')]
435
		);
436
		// setPartial [36457,1 -> 36485,140] [(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35),(36470,38-46),(36473,25-35),(36473,36-51),(36474,28-33),(36474,36-49),(36474,50-58),(36475,35-53),(36475,54-62),(36477,28-32),(36477,33-37),(36477,42-52),(36477,53-69),(36478,32-36),(36478,37-41),(36478,46-56),(36478,57-74),(36479,32-36),(36479,37-41),(36479,46-56),(36479,57-76),(36480,32-36),(36480,37-41),(36480,46-56),(36480,57-68),(36481,32-36),(36481,37-41),(36481,46-56),(36481,57-68),(36482,39-57),(36482,58-66),(36484,34-38),(36484,39-45),(36484,46-50),(36484,55-65),(36484,66-82),(36484,86-97),(36484,98-102),(36484,103-109),(36484,111-124),(36484,125-133),(36485,39-57),(36485,58-66)]
437
		store.setPartial(
438
			new Range(36457, 1, 36485, 140),
439
			[createTokens('[(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35),(36470,38-46),(36473,25-35),(36473,36-51),(36474,28-33),(36474,36-49),(36474,50-58),(36475,35-53),(36475,54-62),(36477,28-32),(36477,33-37),(36477,42-52),(36477,53-69),(36478,32-36),(36478,37-41),(36478,46-56),(36478,57-74),(36479,32-36),(36479,37-41),(36479,46-56),(36479,57-76),(36480,32-36),(36480,37-41),(36480,46-56),(36480,57-68),(36481,32-36),(36481,37-41),(36481,46-56),(36481,57-68),(36482,39-57),(36482,58-66),(36484,34-38),(36484,39-45),(36484,46-50),(36484,55-65),(36484,66-82),(36484,86-97),(36484,98-102),(36484,103-109),(36484,111-124),(36484,125-133),(36485,39-57),(36485,58-66)]')]
440
		);
441
		// setPartial [36441,1 -> 36469,56] [(36442,25-35),(36442,36-50),(36443,30-39),(36443,42-46),(36443,47-53),(36443,54-58),(36443,63-73),(36443,74-84),(36443,87-91),(36443,92-98),(36443,101-105),(36443,106-112),(36443,113-119),(36444,28-37),(36444,38-42),(36444,47-57),(36444,58-75),(36444,80-95),(36444,96-105),(36445,35-53),(36445,54-62),(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35)]
442
		store.setPartial(
443
			new Range(36441, 1, 36469, 56),
444
			[createTokens('[(36442,25-35),(36442,36-50),(36443,30-39),(36443,42-46),(36443,47-53),(36443,54-58),(36443,63-73),(36443,74-84),(36443,87-91),(36443,92-98),(36443,101-105),(36443,106-112),(36443,113-119),(36444,28-37),(36444,38-42),(36444,47-57),(36444,58-75),(36444,80-95),(36444,96-105),(36445,35-53),(36445,54-62),(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35)]')]
445
		);
446

447
		const lineTokens = store.addSparseTokens(36451, new LineTokens(new Uint32Array([60, 1]), `                        if (flags & ModifierFlags.Ambient) {`, codec));
448
		assert.strictEqual(lineTokens.getCount(), 7);
449
	});
450

451

452
	test('issue #95949: Identifiers are colored in bold when targetting keywords', () => {
453

454
		function createTMMetadata(foreground: number, fontStyle: number, languageId: number): number {
455
			return (
456
				(languageId << MetadataConsts.LANGUAGEID_OFFSET)
457
				| (fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
458
				| (foreground << MetadataConsts.FOREGROUND_OFFSET)
459
			) >>> 0;
460
		}
461

462
		function toArr(lineTokens: LineTokens): number[] {
463
			const r: number[] = [];
464
			for (let i = 0; i < lineTokens.getCount(); i++) {
465
				r.push(lineTokens.getEndOffset(i));
466
				r.push(lineTokens.getMetadata(i));
467
			}
468
			return r;
469
		}
470

471
		const codec = new LanguageIdCodec();
472
		const store = new SparseTokensStore(codec);
473

474
		store.set([
475
			SparseMultilineTokens.create(1, new Uint32Array([
476
				0, 6, 11, (1 << MetadataConsts.FOREGROUND_OFFSET) | MetadataConsts.SEMANTIC_USE_FOREGROUND,
477
			]))
478
		], true);
479

480
		const lineTokens = store.addSparseTokens(1, new LineTokens(new Uint32Array([
481
			5, createTMMetadata(5, FontStyle.Bold, 53),
482
			14, createTMMetadata(1, FontStyle.None, 53),
483
			17, createTMMetadata(6, FontStyle.None, 53),
484
			18, createTMMetadata(1, FontStyle.None, 53),
485
		]), `const hello = 123;`, codec));
486

487
		const actual = toArr(lineTokens);
488
		assert.deepStrictEqual(actual, [
489
			5, createTMMetadata(5, FontStyle.Bold, 53),
490
			6, createTMMetadata(1, FontStyle.None, 53),
491
			11, createTMMetadata(1, FontStyle.None, 53),
492
			14, createTMMetadata(1, FontStyle.None, 53),
493
			17, createTMMetadata(6, FontStyle.None, 53),
494
			18, createTMMetadata(1, FontStyle.None, 53)
495
		]);
496
	});
497
});
498

499
Product

Resources

Company
