CoCalc -- convert

GitHub Repository: aos/grafana-agent
Path: blob/main/component/otelcol/exporter/loki/internal/convert/convert_test.go
⁴¹⁰⁰ views
1
package convert_test
2

3
import (
4
	"context"
5
	"testing"
6
	"time"
7

8
	"github.com/grafana/agent/component/common/loki"
9
	"github.com/grafana/agent/component/otelcol/exporter/loki/internal/convert"
10
	"github.com/grafana/agent/pkg/util"
11
	"github.com/prometheus/client_golang/prometheus"
12
	"github.com/stretchr/testify/require"
13
	"go.opentelemetry.io/collector/pdata/plog"
14
)
15

16
func TestConverter(t *testing.T) {
17
	tt := []struct {
18
		name            string
19
		input           string
20
		expectLine      string
21
		expectLabels    string
22
		expectTimestamp time.Time
23
	}{
24
		{
25
			name: "log line without format hint",
26
			input: `{
27
  "resourceLogs": [
28
    {
29
      "resource": {
30
        "attributes": [
31
          {
32
            "key": "host.name",
33
            "value": {
34
              "stringValue": "testHost"
35
            }
36
          }
37
        ],
38
        "droppedAttributesCount": 1
39
      },
40
      "scopeLogs": [
41
        {
42
          "scope": {
43
            "name": "name",
44
            "version": "version",
45
            "droppedAttributesCount": 1
46
          },
47
          "logRecords": [
48
            {
49
              "timeUnixNano": "1672827031972869000",
50
              "observedTimeUnixNano": "1672827031972869000",
51
              "severityNumber": 17,
52
              "severityText": "Error",
53
              "body": {
54
                "stringValue": "hello world"
55
              },
56
              "attributes": [
57
                {
58
                  "key": "sdkVersion",
59
                  "value": {
60
                    "stringValue": "1.0.1"
61
                  }
62
                }
63
              ],
64
              "droppedAttributesCount": 1,
65
              "traceId": "0102030405060708090a0b0c0d0e0f10",
66
              "spanId": "1112131415161718"
67
            }
68
          ],
69
          "schemaUrl": "ScopeLogsSchemaURL"
70
        }
71
      ],
72
      "schemaUrl": "testSchemaURL"
73
    }
74
  ]
75
}`,
76
			expectLine:      `{"body":"hello world","traceid":"0102030405060708090a0b0c0d0e0f10","spanid":"1112131415161718","severity":"Error","attributes":{"sdkVersion":"1.0.1"},"resources":{"host.name":"testHost"}}`,
77
			expectLabels:    `{exporter="OTLP", level="ERROR"}`,
78
			expectTimestamp: time.Date(2023, time.January, 4, 10, 10, 31, 972869000, time.UTC),
79
		},
80
		{
81
			name: "log line with logfmt format hint in resource attributes",
82
			input: `{
83
  "resourceLogs": [
84
    {
85
      "resource": {
86
        "attributes": [
87
          {
88
            "key": "host.name",
89
            "value": {
90
              "stringValue": "testHost"
91
            }
92
          },
93
          {
94
            "key": "loki.format",
95
            "value": {
96
              "stringValue": "logfmt"
97
            }
98
          }
99
        ],
100
        "droppedAttributesCount": 1
101
      },
102
      "scopeLogs": [
103
        {
104
          "scope": {
105
            "name": "name",
106
            "version": "version",
107
            "droppedAttributesCount": 1
108
          },
109
          "logRecords": [
110
            {
111
              "timeUnixNano": "1672827031972869000",
112
              "observedTimeUnixNano": "1672827031972869000",
113
              "severityNumber": 17,
114
              "severityText": "Error",
115
              "body": {
116
                "stringValue": "msg=\"hello world\""
117
              },
118
              "attributes": [
119
                {
120
                  "key": "sdkVersion",
121
                  "value": {
122
                    "stringValue": "1.0.1"
123
                  }
124
                }
125
              ],
126
              "droppedAttributesCount": 1,
127
              "traceId": "0102030405060708090a0b0c0d0e0f10",
128
              "spanId": "1112131415161718"
129
            }
130
          ],
131
          "schemaUrl": "ScopeLogsSchemaURL"
132
        }
133
      ],
134
      "schemaUrl": "testSchemaURL"
135
    }
136
  ]
137
}`,
138
			expectLine:      `msg="hello world" traceID=0102030405060708090a0b0c0d0e0f10 spanID=1112131415161718 severity=Error attribute_sdkVersion=1.0.1 resource_host.name=testHost`,
139
			expectLabels:    `{exporter="OTLP", level="ERROR"}`,
140
			expectTimestamp: time.Date(2023, time.January, 4, 10, 10, 31, 972869000, time.UTC),
141
		},
142
		{
143
			name: "log line with logfmt format hint in log attributes",
144
			input: `{
145
  "resourceLogs": [
146
    {
147
      "resource": {
148
        "attributes": [
149
          {
150
            "key": "host.name",
151
            "value": {
152
              "stringValue": "testHost"
153
            }
154
          }
155
        ],
156
        "droppedAttributesCount": 1
157
      },
158
      "scopeLogs": [
159
        {
160
          "scope": {
161
            "name": "name",
162
            "version": "version",
163
            "droppedAttributesCount": 1
164
          },
165
          "logRecords": [
166
            {
167
              "timeUnixNano": "1672827031972869000",
168
              "observedTimeUnixNano": "1672827031972869000",
169
              "severityNumber": 17,
170
              "severityText": "Error",
171
              "body": {
172
                "stringValue": "msg=\"hello world\""
173
              },
174
              "attributes": [
175
                {
176
                  "key": "sdkVersion",
177
                  "value": {
178
                    "stringValue": "1.0.1"
179
                  }
180
                },
181
                {
182
                  "key": "loki.format",
183
                  "value": {
184
                    "stringValue": "logfmt"
185
                  }
186
                }
187
              ],
188
              "droppedAttributesCount": 1,
189
              "traceId": "0102030405060708090a0b0c0d0e0f10",
190
              "spanId": "1112131415161718"
191
            }
192
          ],
193
          "schemaUrl": "ScopeLogsSchemaURL"
194
        }
195
      ],
196
      "schemaUrl": "testSchemaURL"
197
    }
198
  ]
199
}`,
200
			expectLine:      `msg="hello world" traceID=0102030405060708090a0b0c0d0e0f10 spanID=1112131415161718 severity=Error attribute_sdkVersion=1.0.1 resource_host.name=testHost`,
201
			expectLabels:    `{exporter="OTLP", level="ERROR"}`,
202
			expectTimestamp: time.Date(2023, time.January, 4, 10, 10, 31, 972869000, time.UTC),
203
		},
204
		{
205
			name: "resource attributes converted to labels",
206
			input: `{
207
  "resourceLogs": [
208
    {
209
      "resource": {
210
        "attributes": [
211
          {
212
            "key": "host.name",
213
            "value": {
214
              "stringValue": "testHost"
215
            }
216
          },
217
		  {
218
		    "key": "loki.resource.labels",
219
			"value": {
220
			  "stringValue": "mylabel_1,mylabel_2"
221
			}
222
		  },
223
		  {
224
		    "key": "mylabel_1",
225
			"value": {
226
			  "stringValue": "value_1"
227
			}
228
		  },
229
		  {
230
		    "key": "mylabel_2",
231
			"value": {
232
			  "intValue": "42"
233
			}
234
		  },
235
		  {
236
		    "key": "mylabel_3",
237
			"value": {
238
			  "stringValue": "value_3"
239
			}
240
		  }
241
        ],
242
        "droppedAttributesCount": 1
243
      },
244
      "scopeLogs": [
245
        {
246
          "scope": {
247
            "name": "name",
248
            "version": "version",
249
            "droppedAttributesCount": 1
250
          },
251
          "logRecords": [
252
            {
253
              "timeUnixNano": "1672827031972869000",
254
              "observedTimeUnixNano": "1672827031972869000",
255
              "severityNumber": 17,
256
              "severityText": "Error",
257
              "body": {
258
                "stringValue": "msg=\"hello world\""
259
              },
260
              "attributes": [
261
                {
262
                  "key": "sdkVersion",
263
                  "value": {
264
                    "stringValue": "1.0.1"
265
                  }
266
                },
267
                {
268
                  "key": "loki.format",
269
                  "value": {
270
                    "stringValue": "logfmt"
271
                  }
272
                }
273
              ],
274
              "droppedAttributesCount": 1,
275
              "traceId": "0102030405060708090a0b0c0d0e0f10",
276
              "spanId": "1112131415161718"
277
            }
278
          ],
279
          "schemaUrl": "ScopeLogsSchemaURL"
280
        }
281
      ],
282
      "schemaUrl": "testSchemaURL"
283
    }
284
  ]
285
}`,
286
			expectLine:      `msg="hello world" traceID=0102030405060708090a0b0c0d0e0f10 spanID=1112131415161718 severity=Error attribute_sdkVersion=1.0.1 resource_host.name=testHost resource_mylabel_3=value_3`,
287
			expectLabels:    `{exporter="OTLP", level="ERROR", mylabel_1="value_1", mylabel_2="42"}`,
288
			expectTimestamp: time.Date(2023, time.January, 4, 10, 10, 31, 972869000, time.UTC),
289
		},
290
		{
291
			name: "log attributes converted to labels",
292
			input: `{
293
  "resourceLogs": [
294
    {
295
      "resource": {
296
        "attributes": [
297
          {
298
            "key": "host.name",
299
            "value": {
300
              "stringValue": "testHost"
301
            }
302
          }
303
        ],
304
        "droppedAttributesCount": 1
305
      },
306
      "scopeLogs": [
307
        {
308
          "scope": {
309
            "name": "name",
310
            "version": "version",
311
            "droppedAttributesCount": 1
312
          },
313
          "logRecords": [
314
            {
315
              "timeUnixNano": "1672827031972869000",
316
              "observedTimeUnixNano": "1672827031972869000",
317
              "severityNumber": 17,
318
              "severityText": "Error",
319
              "body": {
320
                "stringValue": "msg=\"hello world\""
321
              },
322
              "attributes": [
323
                {
324
                  "key": "sdkVersion",
325
                  "value": {
326
                    "stringValue": "1.0.1"
327
                  }
328
                },
329
                {
330
                  "key": "loki.format",
331
                  "value": {
332
                    "stringValue": "logfmt"
333
                  }
334
                },
335
		        {
336
		          "key": "loki.attribute.labels",
337
		          "value": {
338
		            "stringValue": "mylabel_1,mylabel_2"
339
		          }
340
		        },
341
		        {
342
		          "key": "mylabel_1",
343
		          "value": {
344
		            "stringValue": "value_1"
345
		          }
346
		        },
347
		        {
348
		          "key": "mylabel_2",
349
		          "value": {
350
		            "intValue": "42"
351
		          }
352
		        },
353
		        {
354
		          "key": "mylabel_3",
355
		          "value": {
356
		            "stringValue": "value_3"
357
		          }
358
		        }
359
              ],
360
              "droppedAttributesCount": 1,
361
              "traceId": "0102030405060708090a0b0c0d0e0f10",
362
              "spanId": "1112131415161718"
363
            }
364
          ],
365
          "schemaUrl": "ScopeLogsSchemaURL"
366
        }
367
      ],
368
      "schemaUrl": "testSchemaURL"
369
    }
370
  ]
371
}`,
372
			expectLine:      `msg="hello world" traceID=0102030405060708090a0b0c0d0e0f10 spanID=1112131415161718 severity=Error attribute_sdkVersion=1.0.1 attribute_mylabel_3=value_3 resource_host.name=testHost`,
373
			expectLabels:    `{exporter="OTLP", level="ERROR", mylabel_1="value_1", mylabel_2="42"}`,
374
			expectTimestamp: time.Date(2023, time.January, 4, 10, 10, 31, 972869000, time.UTC),
375
		},
376
		{
377
			name: "tenant resource attribute converted to label",
378
			input: `{
379
  "resourceLogs": [
380
    {
381
      "resource": {
382
        "attributes": [
383
          {
384
            "key": "host.name",
385
            "value": {
386
              "stringValue": "testHost"
387
            }
388
          }
389
        ],
390
        "droppedAttributesCount": 1
391
      },
392
      "scopeLogs": [
393
        {
394
          "scope": {
395
            "name": "name",
396
            "version": "version",
397
            "droppedAttributesCount": 1
398
          },
399
          "logRecords": [
400
            {
401
              "timeUnixNano": "1672827031972869000",
402
              "observedTimeUnixNano": "1672827031972869000",
403
              "severityNumber": 17,
404
              "severityText": "Error",
405
              "body": {
406
                "stringValue": "hello world"
407
              },
408
              "attributes": [
409
                {
410
                  "key": "sdkVersion",
411
                  "value": {
412
                    "stringValue": "1.0.1"
413
                  }
414
                },
415
                {
416
                  "key": "loki.format",
417
                  "value": {
418
                    "stringValue": "json"
419
                  }
420
                },
421
                {
422
                  "key": "loki.tenant",
423
                  "value": {
424
                    "stringValue": "tenant.id"
425
                  }
426
                },
427
                {
428
                  "key": "tenant.id",
429
                  "value": {
430
                    "stringValue": "tenant_2"
431
                  }
432
                }
433
              ],
434
              "droppedAttributesCount": 1,
435
              "traceId": "0102030405060708090a0b0c0d0e0f10",
436
              "spanId": "1112131415161718"
437
            }
438
          ],
439
          "schemaUrl": "ScopeLogsSchemaURL"
440
        }
441
      ],
442
      "schemaUrl": "testSchemaURL"
443
    }
444
  ]
445
}`,
446
			expectLine:      `{"body":"hello world","traceid":"0102030405060708090a0b0c0d0e0f10","spanid":"1112131415161718","severity":"Error","attributes":{"sdkVersion":"1.0.1"},"resources":{"host.name":"testHost"}}`,
447
			expectLabels:    `{exporter="OTLP", level="ERROR", tenant.id="tenant_2"}`,
448
			expectTimestamp: time.Date(2023, time.January, 4, 10, 10, 31, 972869000, time.UTC),
449
		},
450
	}
451

452
	decoder := &plog.JSONUnmarshaler{}
453
	for _, tc := range tt {
454
		t.Run(tc.name, func(t *testing.T) {
455
			payload, err := decoder.UnmarshalLogs([]byte(tc.input))
456
			require.NoError(t, err)
457

458
			l := util.TestLogger(t)
459
			ch1, ch2 := make(loki.LogsReceiver), make(loki.LogsReceiver)
460
			conv := convert.New(l, prometheus.NewRegistry(), []loki.LogsReceiver{ch1, ch2})
461
			go func() {
462
				require.NoError(t, conv.ConsumeLogs(context.Background(), payload))
463
			}()
464

465
			for i := 0; i < 2; i++ {
466
				select {
467
				case l := <-ch1:
468
					require.Equal(t, tc.expectLine, l.Line)
469
					require.Equal(t, tc.expectLabels, l.Labels.String())
470
					require.Equal(t, tc.expectTimestamp, l.Timestamp.UTC())
471
				case l := <-ch2:
472
					require.Equal(t, tc.expectLine, l.Line)
473
					require.Equal(t, tc.expectLabels, l.Labels.String())
474
					require.Equal(t, tc.expectTimestamp, l.Timestamp.UTC())
475
				case <-time.After(time.Second):
476
					require.FailNow(t, "failed waiting for logs")
477
				}
478
			}
479
		})
480
	}
481
}
482

483
Product

Resources

Company
