CoCalc -- logs_templates

GitHub Repository: aos/grafana-agent
Path: blob/main/pkg/operator/config/logs_templates_test.go
⁴⁰⁹⁵ views
1
package config
2

3
import (
4
	"fmt"
5
	"strings"
6
	"testing"
7

8
	jsonnet "github.com/google/go-jsonnet"
9
	prom "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
10
	prom_v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
11
	"github.com/stretchr/testify/require"
12
	v1 "k8s.io/api/core/v1"
13
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15

16
	gragent "github.com/grafana/agent/pkg/operator/apis/monitoring/v1alpha1"
17
	"github.com/grafana/agent/pkg/operator/assets"
18
	"github.com/grafana/agent/pkg/util"
19
)
20

21
func TestLogsClientConfig(t *testing.T) {
22
	agent := &gragent.GrafanaAgent{
23
		ObjectMeta: metav1.ObjectMeta{
24
			Namespace: "telemetry",
25
			Name:      "grafana-agent",
26
		},
27
	}
28

29
	tt := []struct {
30
		name   string
31
		input  map[string]interface{}
32
		expect string
33
	}{
34
		{
35
			name: "all-in-one URL",
36
			input: map[string]interface{}{
37
				"agent":     agent,
38
				"namespace": "operator",
39
				"spec": &gragent.LogsClientSpec{
40
					URL: "http://username:password@localhost:3100/loki/api/v1/push",
41
				},
42
			},
43
			expect: util.Untab(`
44
				url: http://username:password@localhost:3100/loki/api/v1/push
45
				external_labels:
46
					cluster: telemetry/grafana-agent
47
			`),
48
		},
49
		{
50
			name: "full basic config",
51
			input: map[string]interface{}{
52
				"agent":     agent,
53
				"namespace": "operator",
54
				"spec": &gragent.LogsClientSpec{
55
					URL:       "http://localhost:3100/loki/api/v1/push",
56
					TenantID:  "tenant",
57
					BatchWait: "5m",
58
					BatchSize: 500,
59
					Timeout:   "5m",
60
					ExternalLabels: map[string]string{
61
						"foo":  "bar",
62
						"fizz": "buzz",
63
					},
64
					ProxyURL: "http://proxy:3100/",
65
					BackoffConfig: &gragent.LogsBackoffConfigSpec{
66
						MinPeriod:  "500ms",
67
						MaxPeriod:  "5m",
68
						MaxRetries: 100,
69
					},
70
				},
71
			},
72
			expect: util.Untab(`
73
				url: http://localhost:3100/loki/api/v1/push
74
				tenant_id: tenant
75
				batchwait: 5m
76
				batchsize: 500
77
				proxy_url: http://proxy:3100/
78
				backoff_config:
79
					min_period: 500ms
80
					max_period: 5m
81
					max_retries: 100
82
				external_labels:
83
					cluster: telemetry/grafana-agent
84
					foo: bar
85
					fizz: buzz
86
				timeout: 5m
87
			`),
88
		},
89
		{
90
			name: "tls config",
91
			input: map[string]interface{}{
92
				"agent":     agent,
93
				"namespace": "operator",
94
				"spec": &gragent.LogsClientSpec{
95
					URL: "http://localhost:3100/loki/api/v1/push",
96
					TLSConfig: &prom.TLSConfig{
97
						CAFile:   "ca",
98
						KeyFile:  "key",
99
						CertFile: "cert",
100
					},
101
				},
102
			},
103
			expect: util.Untab(`
104
				url: http://localhost:3100/loki/api/v1/push
105
				tls_config:
106
					ca_file: ca
107
					key_file: key
108
					cert_file: cert
109
				external_labels:
110
					cluster: telemetry/grafana-agent
111
			`),
112
		},
113
		{
114
			name: "bearer tokens",
115
			input: map[string]interface{}{
116
				"agent":     agent,
117
				"namespace": "operator",
118
				"spec": &gragent.LogsClientSpec{
119
					URL:             "http://localhost:3100/loki/api/v1/push",
120
					BearerToken:     "tok",
121
					BearerTokenFile: "tokfile",
122
				},
123
			},
124
			expect: util.Untab(`
125
				url: http://localhost:3100/loki/api/v1/push
126
				bearer_token: tok
127
				bearer_token_file: tokfile
128
				external_labels:
129
					cluster: telemetry/grafana-agent
130
			`),
131
		},
132
		{
133
			name: "basic auth",
134
			input: map[string]interface{}{
135
				"agent":     agent,
136
				"namespace": "operator",
137
				"spec": &gragent.LogsClientSpec{
138
					URL: "http://localhost:3100/loki/api/v1/push",
139
					BasicAuth: &prom.BasicAuth{
140
						Username: v1.SecretKeySelector{
141
							LocalObjectReference: v1.LocalObjectReference{Name: "obj"},
142
							Key:                  "key",
143
						},
144
						Password: v1.SecretKeySelector{
145
							LocalObjectReference: v1.LocalObjectReference{Name: "obj"},
146
							Key:                  "key",
147
						},
148
					},
149
				},
150
			},
151
			expect: util.Untab(`
152
				url: http://localhost:3100/loki/api/v1/push
153
				basic_auth:
154
					username: secretkey
155
					password: secretkey
156
				external_labels:
157
					cluster: telemetry/grafana-agent
158
			`),
159
		},
160
	}
161

162
	for _, tc := range tt {
163
		t.Run(tc.name, func(t *testing.T) {
164
			vm, err := createVM(testStore())
165
			require.NoError(t, err)
166

167
			actual, err := runSnippetTLA(t, vm, "./component/logs/client.libsonnet", tc.input)
168
			require.NoError(t, err)
169
			require.YAMLEq(t, tc.expect, actual)
170
		})
171
	}
172
}
173

174
func TestLogsStages(t *testing.T) {
175
	tt := []struct {
176
		name   string
177
		input  map[string]interface{}
178
		expect string
179
	}{
180
		{
181
			name: "docker",
182
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
183
				Docker: &gragent.DockerStageSpec{},
184
			}},
185
			expect: util.Untab(`docker: {}`),
186
		},
187
		{
188
			name: "cri",
189
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
190
				CRI: &gragent.CRIStageSpec{},
191
			}},
192
			expect: util.Untab(`cri: {}`),
193
		},
194
		{
195
			name: "regex",
196
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
197
				Regex: &gragent.RegexStageSpec{
198
					Source:     "time",
199
					Expression: "^(?P<year>\\d+)",
200
				},
201
			}},
202
			expect: util.Untab(`
203
				regex:
204
					expression: '^(?P<year>\d+)'
205
					source: time
206
			`),
207
		},
208
		{
209
			name: "json",
210
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
211
				JSON: &gragent.JSONStageSpec{
212
					Expressions: map[string]string{"user": ""},
213
					Source:      "extra",
214
				},
215
			}},
216
			expect: util.Untab(`
217
				json:
218
					expressions:
219
						user: ""
220
					source: extra
221
			`),
222
		},
223
		{
224
			name: "labelallow",
225
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
226
				LabelAllow: []string{"foo", "bar"},
227
			}},
228
			expect: util.Untab(`
229
				labelallow: [foo, bar]
230
			`),
231
		},
232
		{
233
			name: "labeldrop",
234
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
235
				LabelDrop: []string{"foo", "bar"},
236
			}},
237
			expect: util.Untab(`
238
				labeldrop: [foo, bar]
239
			`),
240
		},
241
		{
242
			name: "labels",
243
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
244
				Labels: map[string]string{
245
					"foo":  "",
246
					"fizz": "buzz",
247
				},
248
			}},
249
			expect: util.Untab(`
250
				labels:
251
					foo: ""
252
					fizz: buzz
253
			`),
254
		},
255
		{
256
			name: "limit",
257
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
258
				Limit: &gragent.LimitStageSpec{
259
					Rate:  10,
260
					Burst: 20,
261
					Drop:  false,
262
				},
263
			}},
264
			expect: util.Untab(`
265
				limit:
266
					rate: 10
267
					burst: 20
268
					drop: false
269
			`),
270
		},
271
		{
272
			name: "match",
273
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
274
				Match: &gragent.MatchStageSpec{
275
					PipelineName:      "app2",
276
					Selector:          `{app="pokey"}`,
277
					Action:            "keep",
278
					DropCounterReason: "no_pokey",
279
					Stages: util.Untab(`
280
					- json:
281
  			      expressions:
282
							  msg: msg
283
					`),
284
				},
285
			}},
286
			expect: util.Untab(`
287
				match:
288
					pipeline_name: app2
289
					selector: '{app="pokey"}'
290
					action: keep
291
					drop_counter_reason: no_pokey
292
					stages:
293
					- json:
294
							expressions:
295
								msg: msg
296
			`),
297
		},
298
		{
299
			name: "metrics",
300
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
301
				Metrics: map[string]gragent.MetricsStageSpec{
302
					"logs_line_total": {
303
						Type:            "counter",
304
						Description:     "total number of log lines",
305
						Prefix:          "my_promtail_custom_",
306
						MaxIdleDuration: "24h",
307
						MatchAll:        boolPtr(true),
308
						Action:          "inc",
309
					},
310
					"queue_elements": {
311
						Type:        "gauge",
312
						Description: "elements in queue",
313
						Action:      "add",
314
					},
315
					"http_response_time_seconds": {
316
						Type:    "histogram",
317
						Source:  "response_time",
318
						Action:  "inc",
319
						Buckets: []string{"0.001", "0.0025", "0.050"},
320
					},
321
				},
322
			}},
323
			expect: util.Untab(`
324
				metrics:
325
					logs_line_total:
326
						type: Counter
327
						description: total number of log lines
328
						prefix: my_promtail_custom_
329
						max_idle_duration: 24h
330
						config:
331
							match_all: true
332
							action: inc
333
					queue_elements:
334
						type: Gauge
335
						description: elements in queue
336
						config:
337
							action: add
338
					http_response_time_seconds:
339
						type: Histogram
340
						source: response_time
341
						config:
342
							action: inc
343
							buckets: [0.001, 0.0025, 0.050]
344
			`),
345
		},
346
		{
347
			name: "multiline",
348
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
349
				Multiline: &gragent.MultilineStageSpec{
350
					FirstLine:   "first",
351
					MaxWaitTime: "5m",
352
					MaxLines:    5,
353
				},
354
			}},
355
			expect: util.Untab(`
356
				multiline:
357
					firstline: first
358
					max_wait_time: 5m
359
					max_lines: 5
360
			`),
361
		},
362
		{
363
			name: "output",
364
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
365
				Output: &gragent.OutputStageSpec{Source: "message"},
366
			}},
367
			expect: util.Untab(`
368
				output:
369
					source: message
370
			`),
371
		},
372
		{
373
			name: "pack",
374
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
375
				Pack: &gragent.PackStageSpec{
376
					Labels:          []string{"foo", "bar"},
377
					IngestTimestamp: true,
378
				},
379
			}},
380
			expect: util.Untab(`
381
				pack:
382
					labels: [foo, bar]
383
					ingest_timestamp: true
384
			`),
385
		},
386
		{
387
			name: "regex",
388
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
389
				Regex: &gragent.RegexStageSpec{
390
					Source:     "msg",
391
					Expression: "some regex",
392
				},
393
			}},
394
			expect: util.Untab(`
395
				regex:
396
					source: msg
397
					expression: some regex
398
			`),
399
		},
400
		{
401
			name: "replace",
402
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
403
				Replace: &gragent.ReplaceStageSpec{
404
					Expression: "password (\\S+)",
405
					Replace:    "****",
406
					Source:     "msg",
407
				},
408
			}},
409
			expect: util.Untab(`
410
				replace:
411
					expression: 'password (\S+)'
412
					replace: '****'
413
					source: msg
414
			`),
415
		},
416
		{
417
			name: "template",
418
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
419
				Template: &gragent.TemplateStageSpec{
420
					Source:   "new_key",
421
					Template: "hello world!",
422
				},
423
			}},
424
			expect: util.Untab(`
425
				template:
426
					source: new_key
427
					template: "hello world!"
428
			`),
429
		},
430
		{
431
			name: "tenant",
432
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
433
				Tenant: &gragent.TenantStageSpec{
434
					Label:  "__meta_kubernetes_pod_label_fake",
435
					Source: "customer_id",
436
					Value:  "fake",
437
				},
438
			}},
439
			expect: util.Untab(`
440
				tenant:
441
					label: __meta_kubernetes_pod_label_fake
442
					source: customer_id
443
					value: fake
444
			`),
445
		},
446
		{
447
			name: "timestamp",
448
			input: map[string]interface{}{"spec": &gragent.PipelineStageSpec{
449
				Timestamp: &gragent.TimestampStageSpec{
450
					Source:          "time",
451
					Format:          "RFC3339Nano",
452
					FallbackFormats: []string{"UnixNs"},
453
					Location:        "America/New_York",
454
					ActionOnFailure: "fudge",
455
				},
456
			}},
457
			expect: util.Untab(`
458
				timestamp:
459
					source: time
460
					format: RFC3339Nano
461
					fallback_formats: [UnixNs]
462
					location: America/New_York
463
					action_on_failure: fudge
464
			`),
465
		},
466
	}
467

468
	for _, tc := range tt {
469
		t.Run(tc.name, func(t *testing.T) {
470
			vm, err := createVM(testStore())
471
			require.NoError(t, err)
472

473
			actual, err := runSnippetTLA(t, vm, "./component/logs/stages.libsonnet", tc.input)
474
			require.NoError(t, err)
475
			require.YAMLEq(t, tc.expect, actual)
476
		})
477
	}
478
}
479

480
func TestPodLogsConfig(t *testing.T) {
481
	tt := []struct {
482
		name   string
483
		input  map[string]interface{}
484
		expect string
485
	}{
486
		{
487
			name: "default",
488
			input: map[string]interface{}{
489
				"agentNamespace": "operator",
490
				"podLogs": gragent.PodLogs{
491
					ObjectMeta: meta_v1.ObjectMeta{
492
						Namespace: "operator",
493
						Name:      "podlogs",
494
					},
495
					Spec: gragent.PodLogsSpec{
496
						RelabelConfigs: []*prom_v1.RelabelConfig{{
497
							SourceLabels: []prom.LabelName{"input_a", "input_b"},
498
							Separator:    ";",
499
							TargetLabel:  "target_a",
500
							Regex:        "regex",
501
							Modulus:      1234,
502
							Replacement:  "foobar",
503
							Action:       "replace",
504
						}},
505
					},
506
				},
507
				"apiServer":                prom_v1.APIServerConfig{},
508
				"ignoreNamespaceSelectors": false,
509
				"enforcedNamespaceLabel":   "",
510
			},
511
			expect: util.Untab(`
512
				job_name: podLogs/operator/podlogs
513
				kubernetes_sd_configs:
514
				- role: pod
515
				  namespaces:
516
						names: [operator]
517
				relabel_configs:
518
				- source_labels: [job]
519
					target_label: __tmp_prometheus_job_name
520
				- source_labels: [__meta_kubernetes_namespace]
521
					target_label: namespace
522
				- source_labels: [__meta_kubernetes_service_name]
523
					target_label: service
524
				- source_labels: [__meta_kubernetes_pod_name]
525
					target_label: pod
526
				- source_labels: [__meta_kubernetes_pod_container_name]
527
					target_label: container
528
				- target_label: job
529
					replacement: operator/podlogs
530
				- source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name']
531
					target_label: __path__
532
					separator: /
533
					replacement: /var/log/pods/*$1/*.log
534
				- source_labels: ["input_a", "input_b"]
535
					separator: ";"
536
					target_label: "target_a"
537
					regex: regex
538
					modulus: 1234
539
					replacement: foobar
540
					action: replace
541
			`),
542
		},
543
	}
544

545
	for _, tc := range tt {
546
		t.Run(tc.name, func(t *testing.T) {
547
			vm, err := createVM(testStore())
548
			require.NoError(t, err)
549

550
			actual, err := runSnippetTLA(t, vm, "./component/logs/pod_logs.libsonnet", tc.input)
551
			require.NoError(t, err)
552
			require.YAMLEq(t, tc.expect, actual)
553
		})
554
	}
555
}
556

557
func TestLogsConfig(t *testing.T) {
558
	agent := &gragent.GrafanaAgent{
559
		ObjectMeta: metav1.ObjectMeta{
560
			Namespace: "operator",
561
			Name:      "grafana-agent",
562
		},
563
	}
564

565
	tt := []struct {
566
		name   string
567
		input  map[string]interface{}
568
		expect string
569
	}{
570
		{
571
			name: "global clients",
572
			input: map[string]interface{}{
573
				"agent": agent,
574
				"global": &gragent.LogsSubsystemSpec{
575
					Clients: []gragent.LogsClientSpec{{URL: "global"}},
576
				},
577
				"instance": &gragent.LogsDeployment{
578
					Instance: &gragent.LogsInstance{
579
						ObjectMeta: metav1.ObjectMeta{
580
							Namespace: "inst",
581
							Name:      "default",
582
						},
583
						Spec: gragent.LogsInstanceSpec{},
584
					},
585
				},
586
				"apiServer": &prom.APIServerConfig{},
587

588
				"ignoreNamespaceSelectors": false,
589
				"enforcedNamespaceLabel":   "",
590
			},
591
			expect: util.Untab(`
592
				name: inst/default
593
				clients:
594
				- url: global
595
				  external_labels:
596
					  cluster: operator/grafana-agent
597
			`),
598
		},
599
		{
600
			name: "local clients",
601
			input: map[string]interface{}{
602
				"agent": agent,
603
				"global": &gragent.LogsSubsystemSpec{
604
					Clients: []gragent.LogsClientSpec{{URL: "global"}},
605
				},
606
				"instance": &gragent.LogsDeployment{
607
					Instance: &gragent.LogsInstance{
608
						ObjectMeta: metav1.ObjectMeta{
609
							Namespace: "inst",
610
							Name:      "default",
611
						},
612
						Spec: gragent.LogsInstanceSpec{
613
							Clients: []gragent.LogsClientSpec{{URL: "local"}},
614
						},
615
					},
616
				},
617
				"apiServer": &prom.APIServerConfig{},
618

619
				"ignoreNamespaceSelectors": false,
620
				"enforcedNamespaceLabel":   "",
621
			},
622
			expect: util.Untab(`
623
				name: inst/default
624
				clients:
625
				- url: local
626
				  external_labels:
627
					  cluster: operator/grafana-agent
628
			`),
629
		},
630
		{
631
			name: "pod logs",
632
			input: map[string]interface{}{
633
				"agent":  agent,
634
				"global": &gragent.LogsSubsystemSpec{},
635
				"instance": &gragent.LogsDeployment{
636
					Instance: &gragent.LogsInstance{
637
						ObjectMeta: metav1.ObjectMeta{Namespace: "inst", Name: "default"},
638
					},
639
					PodLogs: []*gragent.PodLogs{{
640
						ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "pod"},
641
					}},
642
				},
643
				"apiServer": &prom.APIServerConfig{},
644

645
				"ignoreNamespaceSelectors": false,
646
				"enforcedNamespaceLabel":   "",
647
			},
648
			expect: util.Untab(`
649
				name: inst/default
650
				scrape_configs:
651
				- job_name: podLogs/app/pod
652
					kubernetes_sd_configs:
653
					- namespaces:
654
							names:
655
							- app
656
						role: pod
657
					relabel_configs:
658
					- source_labels:
659
						- job
660
						target_label: __tmp_prometheus_job_name
661
					- source_labels:
662
						- __meta_kubernetes_namespace
663
						target_label: namespace
664
					- source_labels:
665
						- __meta_kubernetes_service_name
666
						target_label: service
667
					- source_labels:
668
						- __meta_kubernetes_pod_name
669
						target_label: pod
670
					- source_labels:
671
						- __meta_kubernetes_pod_container_name
672
						target_label: container
673
					- replacement: app/pod
674
						target_label: job
675
					- source_labels: ['__meta_kubernetes_pod_uid', '__meta_kubernetes_pod_container_name']
676
						target_label: __path__
677
						separator: /
678
						replacement: /var/log/pods/*$1/*.log
679
			`),
680
		},
681
		{
682
			name: "additional scrape configs",
683
			input: map[string]interface{}{
684
				"agent":  agent,
685
				"global": &gragent.LogsSubsystemSpec{},
686
				"instance": &gragent.LogsDeployment{
687
					Instance: &gragent.LogsInstance{
688
						ObjectMeta: metav1.ObjectMeta{Namespace: "inst", Name: "default"},
689
						Spec: gragent.LogsInstanceSpec{
690
							AdditionalScrapeConfigs: &v1.SecretKeySelector{
691
								LocalObjectReference: v1.LocalObjectReference{Name: "additional"},
692
								Key:                  "configs",
693
							},
694
						},
695
					},
696
				},
697
				"apiServer": &prom.APIServerConfig{},
698

699
				"ignoreNamespaceSelectors": false,
700
				"enforcedNamespaceLabel":   "",
701
			},
702
			expect: util.Untab(`
703
				name: inst/default
704
				scrape_configs:
705
					- job_name: extra
706
			`),
707
		},
708
	}
709

710
	for _, tc := range tt {
711
		t.Run(tc.name, func(t *testing.T) {
712
			s := testStore()
713

714
			s[assets.KeyForSecret("inst", &v1.SecretKeySelector{
715
				LocalObjectReference: v1.LocalObjectReference{
716
					Name: "additional",
717
				},
718
				Key: "configs",
719
			})] = `[{ "job_name": "extra" }]`
720

721
			vm, err := createVM(s)
722
			require.NoError(t, err)
723

724
			actual, err := runSnippetTLA(t, vm, "./logs.libsonnet", tc.input)
725
			require.NoError(t, err)
726
			require.YAMLEq(t, tc.expect, actual)
727
		})
728
	}
729
}
730

731
func runSnippetTLA(t *testing.T, vm *jsonnet.VM, filename string, tla map[string]interface{}) (string, error) {
732
	t.Helper()
733

734
	args := make([]string, 0, len(tla))
735
	for arg := range tla {
736
		args = append(args, arg)
737
	}
738

739
	boundArgs := make([]string, len(args))
740
	for i := range args {
741
		boundArgs[i] = fmt.Sprintf("%[1]s=%[1]s", args[i])
742
	}
743

744
	// Bind argument to TLA.
745
	for arg, value := range tla {
746
		bb, err := jsonnetMarshal(value)
747
		require.NoError(t, err)
748
		vm.TLACode(arg, string(bb))
749
	}
750

751
	return vm.EvaluateAnonymousSnippet(
752
		filename,
753
		fmt.Sprintf(`
754
			local marshal = import './ext/marshal.libsonnet';
755
			local optionals = import './ext/optionals.libsonnet';
756
			local eval = import '%s';
757
			function(%s) marshal.YAML(optionals.trim(eval(%s)))
758
		`, filename, strings.Join(args, ","), strings.Join(boundArgs, ",")),
759
	)
760
}
761

762
func boolPtr(v bool) *bool { return &v }
763

764
Product

Resources

Company
