Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
microsoft
GitHub Repository: microsoft/vscode
Path: blob/main/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts
13399 views
1
/*---------------------------------------------------------------------------------------------
2
* Copyright (c) Microsoft Corporation. All rights reserved.
3
* Licensed under the MIT License. See License.txt in the project root for license information.
4
*--------------------------------------------------------------------------------------------*/
5
6
import { ChatFetchError } from '../../../platform/chat/common/commonTypes';
7
import { isAutoModel } from '../../../platform/endpoint/node/autoChatEndpoint';
8
import { FetcherId } from '../../../platform/networking/common/fetcherService';
9
import { IChatEndpoint, IChatRequestTelemetryProperties, IEndpointBody } from '../../../platform/networking/common/networking';
10
import { ChatCompletion } from '../../../platform/networking/common/openai';
11
import { ITelemetryService } from '../../../platform/telemetry/common/telemetry';
12
import { TelemetryData } from '../../../platform/telemetry/common/telemetryData';
13
import { isBYOKModel } from '../../byok/node/openAIEndpoint';
14
15
export interface IChatMLFetcherSuccessfulData {
16
chatCompletion: ChatCompletion;
17
baseTelemetry: TelemetryData;
18
userInitiatedRequest: boolean | undefined;
19
chatEndpointInfo: IChatEndpoint | undefined;
20
requestBody: IEndpointBody;
21
maxResponseTokens: number;
22
promptTokenCount: number;
23
timeToFirstToken: number;
24
timeToFirstTokenEmitted: number;
25
hasImageMessages: boolean;
26
transport: string;
27
fetcher: FetcherId | undefined;
28
bytesReceived: number | undefined;
29
suspendEventSeen: boolean | undefined;
30
resumeEventSeen: boolean | undefined;
31
}
32
33
export interface IChatMLFetcherCancellationProperties {
34
source: string;
35
requestId: string;
36
model: string;
37
apiType: string | undefined;
38
transport: string;
39
associatedRequestId?: string;
40
retryAfterError?: string;
41
retryAfterErrorGitHubRequestId?: string;
42
connectivityTestError?: string;
43
connectivityTestErrorGitHubRequestId?: string;
44
retryAfterFilterCategory?: string;
45
fetcher: FetcherId | undefined;
46
suspendEventSeen: boolean | undefined;
47
resumeEventSeen: boolean | undefined;
48
}
49
50
export interface IChatMLFetcherCancellationMeasures {
51
totalTokenMax: number;
52
promptTokenCount: number;
53
tokenCountMax: number;
54
timeToFirstToken: number | undefined;
55
timeToFirstTokenEmitted?: number;
56
timeToCancelled: number;
57
isVisionRequest: number;
58
isBYOK: number;
59
isAuto: number;
60
bytesReceived: number | undefined;
61
issuedTime: number;
62
}
63
64
export interface IChatMLFetcherErrorData {
65
processed: ChatFetchError;
66
telemetryProperties: IChatRequestTelemetryProperties | undefined;
67
chatEndpointInfo: IChatEndpoint;
68
requestBody: IEndpointBody;
69
tokenCount: number;
70
maxResponseTokens: number;
71
timeToFirstToken: number;
72
isVisionRequest: boolean;
73
transport: string;
74
fetcher: FetcherId | undefined;
75
bytesReceived: number | undefined;
76
issuedTime: number;
77
wasRetried: boolean;
78
suspendEventSeen: boolean | undefined;
79
resumeEventSeen: boolean | undefined;
80
}
81
82
export class ChatMLFetcherTelemetrySender {
83
84
public static sendSuccessTelemetry(
85
telemetryService: ITelemetryService,
86
{
87
chatCompletion,
88
baseTelemetry,
89
userInitiatedRequest,
90
chatEndpointInfo,
91
requestBody,
92
maxResponseTokens,
93
promptTokenCount,
94
timeToFirstToken,
95
timeToFirstTokenEmitted,
96
hasImageMessages,
97
transport,
98
fetcher,
99
bytesReceived,
100
suspendEventSeen,
101
resumeEventSeen,
102
}: IChatMLFetcherSuccessfulData,
103
) {
104
/* __GDPR__
105
"response.success" : {
106
"owner": "digitarald",
107
"comment": "Report quality details for a successful service response.",
108
"reason": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reason for why a response finished" },
109
"filterReason": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reason for why a response was filtered" },
110
"source": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Source of the initial request" },
111
"initiatorType": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was initiated by a user or an agent" },
112
"model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" },
113
"modelInvoked": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Actual model invoked for the response" },
114
"apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" },
115
"conversationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Id for the current chat conversation." },
116
"requestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Id of the current turn request" },
117
"gitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id if available" },
118
"associatedRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Another request ID that this request is associated with (eg, the originating request of a summarization request)." },
119
"reasoningEffort": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reasoning effort level" },
120
"reasoningSummary": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reasoning summary level" },
121
"fetcher": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The fetcher used for the request" },
122
"transport": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The transport used for the request (http or websocket)" },
123
"totalTokenMax": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Maximum total token window", "isMeasurement": true },
124
"clientPromptTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens, locally counted", "isMeasurement": true },
125
"promptTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens, server side counted", "isMeasurement": true },
126
"promptCacheTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens hitting cache as reported by server", "isMeasurement": true },
127
"tokenCountMax": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Maximum generated tokens", "isMeasurement": true },
128
"tokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of generated tokens", "isMeasurement": true },
129
"reasoningTokens": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of reasoning tokens", "isMeasurement": true },
130
"acceptedPredictionTokens": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Number of tokens in the prediction that appeared in the completion", "isMeasurement": true },
131
"rejectedPredictionTokens": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Number of tokens in the prediction that appeared in the completion", "isMeasurement": true },
132
"completionTokens": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Number of tokens in the output", "isMeasurement": true },
133
"timeToFirstToken": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token", "isMeasurement": true },
134
"timeToFirstTokenEmitted": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token emitted (visible text)", "isMeasurement": true },
135
"timeToComplete": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to complete the request", "isMeasurement": true },
136
"issuedTime": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Timestamp when the request was issued", "isMeasurement": true },
137
"isVisionRequest": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the request was for a vision model", "isMeasurement": true },
138
"isBYOK": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for a BYOK model", "isMeasurement": true },
139
"isAuto": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for an Auto model", "isMeasurement": true },
140
"bytesReceived": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of bytes received in the response", "isMeasurement": true },
141
"retryAfterError": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error of the original request." },
142
"retryAfterErrorGitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id of the original request if available" },
143
"connectivityTestError": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error of the connectivity test." },
144
"connectivityTestErrorGitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id of the connectivity test request if available" },
145
"retryAfterFilterCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response was filtered and this is a retry attempt, this contains the original filtered content category." },
146
"suspendEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system suspend event was seen during the request", "isMeasurement": true },
147
"resumeEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system resume event was seen during the request", "isMeasurement": true }
148
}
149
*/
150
telemetryService.sendTelemetryEvent('response.success', { github: true, microsoft: true }, {
151
reason: chatCompletion.finishReason,
152
filterReason: chatCompletion.filterReason,
153
source: baseTelemetry?.properties.messageSource ?? 'unknown',
154
initiatorType: userInitiatedRequest ? 'user' : 'agent',
155
conversationId: baseTelemetry?.properties.conversationId,
156
model: chatEndpointInfo?.model,
157
modelInvoked: chatCompletion.model,
158
apiType: chatEndpointInfo?.apiType,
159
requestId: chatCompletion.requestId.headerRequestId,
160
gitHubRequestId: chatCompletion.requestId.gitHubRequestId,
161
associatedRequestId: baseTelemetry?.properties.associatedRequestId,
162
reasoningEffort: requestBody.reasoning?.effort ?? requestBody.output_config?.effort,
163
reasoningSummary: requestBody.reasoning?.summary,
164
...(fetcher ? { fetcher } : {}),
165
transport,
166
...(baseTelemetry?.properties.retryAfterError ? { retryAfterError: baseTelemetry.properties.retryAfterError } : {}),
167
...(baseTelemetry?.properties.retryAfterErrorGitHubRequestId ? { retryAfterErrorGitHubRequestId: baseTelemetry.properties.retryAfterErrorGitHubRequestId } : {}),
168
...(baseTelemetry?.properties.connectivityTestError ? { connectivityTestError: baseTelemetry.properties.connectivityTestError } : {}),
169
...(baseTelemetry?.properties.connectivityTestErrorGitHubRequestId ? { connectivityTestErrorGitHubRequestId: baseTelemetry.properties.connectivityTestErrorGitHubRequestId } : {}),
170
...(baseTelemetry?.properties.retryAfterFilterCategory ? { retryAfterFilterCategory: baseTelemetry.properties.retryAfterFilterCategory } : {}),
171
}, {
172
totalTokenMax: chatEndpointInfo?.modelMaxPromptTokens ?? -1,
173
tokenCountMax: maxResponseTokens,
174
promptTokenCount: chatCompletion.usage?.prompt_tokens,
175
promptCacheTokenCount: chatCompletion.usage?.prompt_tokens_details?.cached_tokens,
176
clientPromptTokenCount: promptTokenCount,
177
tokenCount: chatCompletion.usage?.total_tokens,
178
reasoningTokens: chatCompletion.usage?.completion_tokens_details?.reasoning_tokens,
179
acceptedPredictionTokens: chatCompletion.usage?.completion_tokens_details?.accepted_prediction_tokens,
180
rejectedPredictionTokens: chatCompletion.usage?.completion_tokens_details?.rejected_prediction_tokens,
181
completionTokens: chatCompletion.usage?.completion_tokens,
182
timeToFirstToken,
183
timeToFirstTokenEmitted,
184
timeToComplete: Date.now() - baseTelemetry.issuedTime,
185
issuedTime: baseTelemetry.issuedTime,
186
isVisionRequest: hasImageMessages ? 1 : -1,
187
isBYOK: isBYOKModel(chatEndpointInfo),
188
isAuto: isAutoModel(chatEndpointInfo),
189
bytesReceived,
190
suspendEventSeen: suspendEventSeen ? 1 : 0,
191
resumeEventSeen: resumeEventSeen ? 1 : 0,
192
});
193
}
194
195
public static sendCancellationTelemetry(
196
telemetryService: ITelemetryService,
197
{
198
source,
199
requestId,
200
model,
201
apiType,
202
transport,
203
associatedRequestId,
204
retryAfterError,
205
retryAfterErrorGitHubRequestId,
206
connectivityTestError,
207
connectivityTestErrorGitHubRequestId,
208
retryAfterFilterCategory,
209
fetcher,
210
suspendEventSeen,
211
resumeEventSeen,
212
}: IChatMLFetcherCancellationProperties,
213
{
214
totalTokenMax,
215
promptTokenCount,
216
tokenCountMax,
217
timeToFirstToken,
218
timeToFirstTokenEmitted,
219
timeToCancelled,
220
isVisionRequest,
221
isBYOK,
222
isAuto,
223
bytesReceived,
224
issuedTime,
225
}: IChatMLFetcherCancellationMeasures
226
) {
227
/* __GDPR__
228
"response.cancelled" : {
229
"owner": "digitarald",
230
"comment": "Report canceled service responses for quality.",
231
"model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" },
232
"apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" },
233
"source": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Source for why the request was made" },
234
"requestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Id of the request" },
235
"associatedRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Another request ID that this request is associated with (eg, the originating request of a summarization request)." },
236
"fetcher": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The fetcher used for the request" },
237
"transport": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The transport used for the request (http or websocket)" },
238
"totalTokenMax": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Maximum total token window", "isMeasurement": true },
239
"promptTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens", "isMeasurement": true },
240
"tokenCountMax": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Maximum generated tokens", "isMeasurement": true },
241
"timeToFirstToken": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token", "isMeasurement": true },
242
"timeToFirstTokenEmitted": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token emitted (visible text)", "isMeasurement": true },
243
"timeToCancelled": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to cancellation", "isMeasurement": true },
244
"timeToComplete": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to complete the request", "isMeasurement": true },
245
"issuedTime": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Timestamp when the request was issued", "isMeasurement": true },
246
"isVisionRequest": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the request was for a vision model", "isMeasurement": true },
247
"isBYOK": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for a BYOK model", "isMeasurement": true },
248
"isAuto": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for an Auto model", "isMeasurement": true },
249
"bytesReceived": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of bytes received before cancellation", "isMeasurement": true },
250
"retryAfterError": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error of the original request." },
251
"retryAfterErrorGitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id of the original request if available" },
252
"connectivityTestError": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error of the connectivity test." },
253
"connectivityTestErrorGitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id of the connectivity test request if available" },
254
"retryAfterFilterCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response was filtered and this is a retry attempt, this contains the original filtered content category." },
255
"suspendEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system suspend event was seen during the request", "isMeasurement": true },
256
"resumeEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system resume event was seen during the request", "isMeasurement": true }
257
}
258
*/
259
telemetryService.sendTelemetryEvent('response.cancelled', { github: true, microsoft: true }, {
260
apiType,
261
source,
262
requestId,
263
model,
264
associatedRequestId,
265
...(fetcher ? { fetcher } : {}),
266
transport,
267
...(retryAfterError ? { retryAfterError } : {}),
268
...(retryAfterErrorGitHubRequestId ? { retryAfterErrorGitHubRequestId } : {}),
269
...(connectivityTestError ? { connectivityTestError } : {}),
270
...(connectivityTestErrorGitHubRequestId ? { connectivityTestErrorGitHubRequestId } : {}),
271
...(retryAfterFilterCategory ? { retryAfterFilterCategory } : {})
272
}, {
273
totalTokenMax,
274
promptTokenCount,
275
tokenCountMax,
276
timeToFirstToken,
277
timeToFirstTokenEmitted,
278
timeToCancelled,
279
timeToComplete: timeToCancelled,
280
issuedTime,
281
isVisionRequest,
282
isBYOK,
283
isAuto,
284
bytesReceived,
285
suspendEventSeen: suspendEventSeen ? 1 : 0,
286
resumeEventSeen: resumeEventSeen ? 1 : 0,
287
});
288
}
289
290
public static sendResponseErrorTelemetry(
291
telemetryService: ITelemetryService,
292
{
293
processed,
294
telemetryProperties,
295
chatEndpointInfo,
296
requestBody,
297
tokenCount,
298
maxResponseTokens,
299
timeToFirstToken,
300
isVisionRequest,
301
transport,
302
fetcher,
303
bytesReceived,
304
issuedTime,
305
wasRetried,
306
suspendEventSeen,
307
resumeEventSeen,
308
}: IChatMLFetcherErrorData,
309
) {
310
/* __GDPR__
311
"response.error" : {
312
"owner": "digitarald",
313
"comment": "Report quality issue for when a service response failed.",
314
"type": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Type of issue" },
315
"reason": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reason of issue" },
316
"model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" },
317
"apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" },
318
"source": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Source for why the request was made" },
319
"requestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Id of the request" },
320
"gitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id if available" },
321
"associatedRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Another request ID that this request is associated with (eg, the originating request of a summarization request)." },
322
"reasoningEffort": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reasoning effort level" },
323
"reasoningSummary": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reasoning summary level" },
324
"fetcher": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The fetcher used for the request" },
325
"transport": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The transport used for the request (http or websocket)" },
326
"totalTokenMax": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Maximum total token window", "isMeasurement": true },
327
"promptTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens", "isMeasurement": true },
328
"tokenCountMax": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Maximum generated tokens", "isMeasurement": true },
329
"timeToFirstToken": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token", "isMeasurement": true },
330
"timeToFirstTokenEmitted": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to first token emitted (visible text)", "isMeasurement": true },
331
"timeToComplete": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Time to complete the request", "isMeasurement": true },
332
"issuedTime": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Timestamp when the request was issued", "isMeasurement": true },
333
"isVisionRequest": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the request was for a vision model", "isMeasurement": true },
334
"isBYOK": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for a BYOK model", "isMeasurement": true },
335
"isAuto": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the request was for an Auto model", "isMeasurement": true },
336
"wasRetried": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether the error will be retried", "isMeasurement": true },
337
"bytesReceived": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of bytes received before the error", "isMeasurement": true },
338
"retryAfterError": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error of the original request." },
339
"retryAfterErrorGitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id of the original request if available" },
340
"connectivityTestError": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Error of the connectivity test." },
341
"connectivityTestErrorGitHubRequestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "GitHub request id of the connectivity test request if available" },
342
"retryAfterFilterCategory": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "If the response was filtered and this is a retry attempt, this contains the original filtered content category." },
343
"suspendEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system suspend event was seen during the request", "isMeasurement": true },
344
"resumeEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system resume event was seen during the request", "isMeasurement": true }
345
}
346
*/
347
telemetryService.sendTelemetryEvent('response.error', { github: true, microsoft: true }, {
348
type: processed.type,
349
reason: processed.reasonDetail || processed.reason,
350
source: telemetryProperties?.messageSource ?? 'unknown',
351
requestId: processed.requestId,
352
gitHubRequestId: processed.serverRequestId,
353
model: chatEndpointInfo.model,
354
apiType: chatEndpointInfo.apiType,
355
reasoningEffort: requestBody.reasoning?.effort ?? requestBody.output_config?.effort,
356
reasoningSummary: requestBody.reasoning?.summary,
357
...(fetcher ? { fetcher } : {}),
358
transport,
359
associatedRequestId: telemetryProperties?.associatedRequestId,
360
...(telemetryProperties?.retryAfterError ? { retryAfterError: telemetryProperties.retryAfterError } : {}),
361
...(telemetryProperties?.retryAfterErrorGitHubRequestId ? { retryAfterErrorGitHubRequestId: telemetryProperties.retryAfterErrorGitHubRequestId } : {}),
362
...(telemetryProperties?.connectivityTestError ? { connectivityTestError: telemetryProperties.connectivityTestError } : {}),
363
...(telemetryProperties?.connectivityTestErrorGitHubRequestId ? { connectivityTestErrorGitHubRequestId: telemetryProperties.connectivityTestErrorGitHubRequestId } : {}),
364
...(telemetryProperties?.retryAfterFilterCategory ? { retryAfterFilterCategory: telemetryProperties.retryAfterFilterCategory } : {})
365
}, {
366
totalTokenMax: chatEndpointInfo.modelMaxPromptTokens ?? -1,
367
promptTokenCount: tokenCount,
368
tokenCountMax: maxResponseTokens,
369
timeToFirstToken,
370
timeToComplete: Date.now() - issuedTime,
371
issuedTime,
372
isVisionRequest: isVisionRequest ? 1 : -1,
373
isBYOK: isBYOKModel(chatEndpointInfo),
374
isAuto: isAutoModel(chatEndpointInfo),
375
wasRetried: wasRetried ? 1 : 0,
376
bytesReceived,
377
suspendEventSeen: suspendEventSeen ? 1 : 0,
378
resumeEventSeen: resumeEventSeen ? 1 : 0,
379
});
380
}
381
}
382
383