CoCalc -- ws-daemon.json

GitHub Repository: gitpod-io/gitpod
Path: blob/main/operations/observability/mixins/workspace/dashboards/components/ws-daemon.json
²⁵⁰⁶ views
1
{
2
  "annotations": {
3
    "list": [
4
      {
5
        "builtIn": 1,
6
        "datasource": {
7
          "type": "datasource",
8
          "uid": "grafana"
9
        },
10
        "enable": true,
11
        "hide": true,
12
        "iconColor": "rgba(0, 211, 255, 1)",
13
        "name": "Annotations & Alerts",
14
        "target": {
15
          "limit": 100,
16
          "matchAny": false,
17
          "tags": [],
18
          "type": "dashboard"
19
        },
20
        "type": "dashboard"
21
      }
22
    ]
23
  },
24
  "editable": true,
25
  "fiscalYearStartMonth": 0,
26
  "graphTooltip": 1,
27
  "id": null,
28
  "links": [],
29
  "liveNow": false,
30
  "panels": [
31
    {
32
      "collapsed": false,
33
      "gridPos": {
34
        "h": 1,
35
        "w": 24,
36
        "x": 0,
37
        "y": 0
38
      },
39
      "id": 65,
40
      "panels": [],
41
      "title": "Controller Manager",
42
      "type": "row"
43
    },
44
    {
45
      "aliasColors": {},
46
      "bars": false,
47
      "dashLength": 10,
48
      "dashes": false,
49
      "datasource": {
50
        "type": "prometheus",
51
        "uid": "$datasource"
52
      },
53
      "fieldConfig": {
54
        "defaults": {
55
          "links": []
56
        },
57
        "overrides": []
58
      },
59
      "fill": 1,
60
      "fillGradient": 0,
61
      "gridPos": {
62
        "h": 9,
63
        "w": 8,
64
        "x": 0,
65
        "y": 1
66
      },
67
      "hiddenSeries": false,
68
      "id": 67,
69
      "legend": {
70
        "alignAsTable": true,
71
        "avg": false,
72
        "current": true,
73
        "max": false,
74
        "min": false,
75
        "rightSide": false,
76
        "show": true,
77
        "total": false,
78
        "values": true
79
      },
80
      "lines": true,
81
      "linewidth": 1,
82
      "links": [],
83
      "nullPointMode": "null",
84
      "options": {
85
        "alertThreshold": true
86
      },
87
      "percentage": false,
88
      "pluginVersion": "9.5.2",
89
      "pointradius": 5,
90
      "points": false,
91
      "renderer": "flot",
92
      "seriesOverrides": [],
93
      "spaceLength": 10,
94
      "stack": false,
95
      "steppedLine": false,
96
      "targets": [
97
        {
98
          "datasource": {
99
            "uid": "$datasource"
100
          },
101
          "editorMode": "code",
102
          "expr": "topk(20, sum(rate(workqueue_adds_total{job=\"ws-daemon\", pod=~\"$pod\", name=~\"$controller\"}[5m])) by (cluster, pod, name))",
103
          "format": "time_series",
104
          "intervalFactor": 2,
105
          "legendFormat": "{{cluster}} - {{pod}} - {{name}}",
106
          "range": true,
107
          "refId": "A"
108
        }
109
      ],
110
      "thresholds": [],
111
      "timeRegions": [],
112
      "title": "Work Queue Add Rate",
113
      "tooltip": {
114
        "shared": false,
115
        "sort": 0,
116
        "value_type": "individual"
117
      },
118
      "type": "graph",
119
      "xaxis": {
120
        "mode": "time",
121
        "show": true,
122
        "values": []
123
      },
124
      "yaxes": [
125
        {
126
          "$$hashKey": "object:123",
127
          "format": "ops",
128
          "logBase": 1,
129
          "show": true
130
        },
131
        {
132
          "$$hashKey": "object:124",
133
          "format": "ops",
134
          "logBase": 1,
135
          "show": true
136
        }
137
      ],
138
      "yaxis": {
139
        "align": false
140
      }
141
    },
142
    {
143
      "aliasColors": {},
144
      "bars": false,
145
      "dashLength": 10,
146
      "dashes": false,
147
      "datasource": {
148
        "type": "prometheus",
149
        "uid": "$datasource"
150
      },
151
      "fieldConfig": {
152
        "defaults": {
153
          "links": []
154
        },
155
        "overrides": []
156
      },
157
      "fill": 1,
158
      "fillGradient": 0,
159
      "gridPos": {
160
        "h": 9,
161
        "w": 8,
162
        "x": 8,
163
        "y": 1
164
      },
165
      "hiddenSeries": false,
166
      "id": 69,
167
      "legend": {
168
        "alignAsTable": true,
169
        "avg": false,
170
        "current": true,
171
        "max": false,
172
        "min": false,
173
        "rightSide": false,
174
        "show": true,
175
        "total": false,
176
        "values": true
177
      },
178
      "lines": true,
179
      "linewidth": 1,
180
      "links": [],
181
      "nullPointMode": "null",
182
      "options": {
183
        "alertThreshold": true
184
      },
185
      "percentage": false,
186
      "pluginVersion": "9.5.2",
187
      "pointradius": 5,
188
      "points": false,
189
      "renderer": "flot",
190
      "seriesOverrides": [],
191
      "spaceLength": 10,
192
      "stack": false,
193
      "steppedLine": false,
194
      "targets": [
195
        {
196
          "datasource": {
197
            "uid": "$datasource"
198
          },
199
          "editorMode": "code",
200
          "expr": "topk(20, sum(workqueue_depth{job=\"ws-daemon\", cluster=~\"$cluster\", name=~\"$controller\"}) by (cluster, pod, name))",
201
          "format": "time_series",
202
          "intervalFactor": 2,
203
          "legendFormat": "{{cluster}} - {{pod}} - {{name}}",
204
          "range": true,
205
          "refId": "A"
206
        }
207
      ],
208
      "thresholds": [],
209
      "timeRegions": [],
210
      "title": "Work Queue Depth",
211
      "tooltip": {
212
        "shared": false,
213
        "sort": 0,
214
        "value_type": "individual"
215
      },
216
      "type": "graph",
217
      "xaxis": {
218
        "mode": "time",
219
        "show": true,
220
        "values": []
221
      },
222
      "yaxes": [
223
        {
224
          "$$hashKey": "object:123",
225
          "format": "none",
226
          "logBase": 1,
227
          "min": "0",
228
          "show": true
229
        },
230
        {
231
          "$$hashKey": "object:124",
232
          "format": "ops",
233
          "logBase": 1,
234
          "show": true
235
        }
236
      ],
237
      "yaxis": {
238
        "align": false
239
      }
240
    },
241
    {
242
      "datasource": {
243
        "type": "prometheus",
244
        "uid": "$datasource"
245
      },
246
      "fieldConfig": {
247
        "defaults": {
248
          "custom": {
249
            "hideFrom": {
250
              "legend": false,
251
              "tooltip": false,
252
              "viz": false
253
            },
254
            "scaleDistribution": {
255
              "type": "linear"
256
            }
257
          },
258
          "links": []
259
        },
260
        "overrides": []
261
      },
262
      "gridPos": {
263
        "h": 9,
264
        "w": 8,
265
        "x": 16,
266
        "y": 1
267
      },
268
      "id": 71,
269
      "links": [],
270
      "options": {
271
        "calculate": false,
272
        "cellGap": 1,
273
        "color": {
274
          "exponent": 0.5,
275
          "fill": "dark-orange",
276
          "mode": "scheme",
277
          "reverse": false,
278
          "scale": "exponential",
279
          "scheme": "Spectral",
280
          "steps": 64
281
        },
282
        "exemplars": {
283
          "color": "rgba(255,0,255,0.7)"
284
        },
285
        "filterValues": {
286
          "le": 1e-9
287
        },
288
        "legend": {
289
          "show": true
290
        },
291
        "rowsFrame": {
292
          "layout": "auto"
293
        },
294
        "tooltip": {
295
          "show": true,
296
          "yHistogram": false
297
        },
298
        "yAxis": {
299
          "axisPlacement": "left",
300
          "reverse": false,
301
          "unit": "s"
302
        }
303
      },
304
      "pluginVersion": "9.5.2",
305
      "targets": [
306
        {
307
          "datasource": {
308
            "uid": "$datasource"
309
          },
310
          "editorMode": "code",
311
          "expr": "sum(rate(workqueue_queue_duration_seconds_bucket{job=\"ws-daemon\", pod=~\"$pod\", name=~\"$controller\"}[$__rate_interval])) by (le)",
312
          "format": "heatmap",
313
          "intervalFactor": 2,
314
          "legendFormat": "__auto",
315
          "range": true,
316
          "refId": "A"
317
        }
318
      ],
319
      "title": "Work Queue Latency",
320
      "type": "heatmap"
321
    },
322
    {
323
      "aliasColors": {},
324
      "bars": false,
325
      "dashLength": 10,
326
      "dashes": false,
327
      "datasource": {
328
        "type": "prometheus",
329
        "uid": "$datasource"
330
      },
331
      "fieldConfig": {
332
        "defaults": {
333
          "links": []
334
        },
335
        "overrides": []
336
      },
337
      "fill": 1,
338
      "fillGradient": 0,
339
      "gridPos": {
340
        "h": 9,
341
        "w": 8,
342
        "x": 0,
343
        "y": 10
344
      },
345
      "hiddenSeries": false,
346
      "id": 73,
347
      "legend": {
348
        "alignAsTable": false,
349
        "avg": false,
350
        "current": false,
351
        "max": false,
352
        "min": false,
353
        "rightSide": false,
354
        "show": true,
355
        "total": false,
356
        "values": false
357
      },
358
      "lines": true,
359
      "linewidth": 1,
360
      "links": [],
361
      "nullPointMode": "null",
362
      "options": {
363
        "alertThreshold": true
364
      },
365
      "percentage": false,
366
      "pluginVersion": "9.5.2",
367
      "pointradius": 5,
368
      "points": false,
369
      "renderer": "flot",
370
      "seriesOverrides": [],
371
      "spaceLength": 10,
372
      "stack": false,
373
      "steppedLine": false,
374
      "targets": [
375
        {
376
          "datasource": {
377
            "uid": "$datasource"
378
          },
379
          "editorMode": "code",
380
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-daemon\", pod=~\"$pod\",code=~\"2..\"}[5m]))",
381
          "format": "time_series",
382
          "intervalFactor": 2,
383
          "legendFormat": "2xx",
384
          "range": true,
385
          "refId": "A"
386
        },
387
        {
388
          "datasource": {
389
            "uid": "$datasource"
390
          },
391
          "editorMode": "code",
392
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-daemon\", pod=~\"$pod\",code=~\"3..\"}[5m]))",
393
          "format": "time_series",
394
          "intervalFactor": 2,
395
          "legendFormat": "3xx",
396
          "range": true,
397
          "refId": "B"
398
        },
399
        {
400
          "datasource": {
401
            "uid": "$datasource"
402
          },
403
          "editorMode": "code",
404
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-daemon\", pod=~\"$pod\",code=~\"4..\"}[5m]))",
405
          "format": "time_series",
406
          "intervalFactor": 2,
407
          "legendFormat": "4xx",
408
          "range": true,
409
          "refId": "C"
410
        },
411
        {
412
          "datasource": {
413
            "uid": "$datasource"
414
          },
415
          "editorMode": "code",
416
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-daemon\", pod=~\"$pod\",code=~\"5..\"}[5m]))",
417
          "format": "time_series",
418
          "intervalFactor": 2,
419
          "legendFormat": "5xx",
420
          "range": true,
421
          "refId": "D"
422
        }
423
      ],
424
      "thresholds": [],
425
      "timeRegions": [],
426
      "title": "Kube API Request Rate",
427
      "tooltip": {
428
        "shared": false,
429
        "sort": 0,
430
        "value_type": "individual"
431
      },
432
      "type": "graph",
433
      "xaxis": {
434
        "mode": "time",
435
        "show": true,
436
        "values": []
437
      },
438
      "yaxes": [
439
        {
440
          "$$hashKey": "object:705",
441
          "format": "ops",
442
          "logBase": 1,
443
          "show": true
444
        },
445
        {
446
          "$$hashKey": "object:706",
447
          "format": "ops",
448
          "logBase": 1,
449
          "show": true
450
        }
451
      ],
452
      "yaxis": {
453
        "align": false
454
      }
455
    },
456
    {
457
      "datasource": {
458
        "type": "prometheus",
459
        "uid": "$datasource"
460
      },
461
      "fieldConfig": {
462
        "defaults": {
463
          "color": {
464
            "mode": "palette-classic"
465
          },
466
          "custom": {
467
            "axisCenteredZero": false,
468
            "axisColorMode": "text",
469
            "axisLabel": "",
470
            "axisPlacement": "auto",
471
            "barAlignment": 0,
472
            "drawStyle": "line",
473
            "fillOpacity": 10,
474
            "gradientMode": "none",
475
            "hideFrom": {
476
              "legend": false,
477
              "tooltip": false,
478
              "viz": false
479
            },
480
            "lineInterpolation": "linear",
481
            "lineWidth": 1,
482
            "pointSize": 5,
483
            "scaleDistribution": {
484
              "type": "linear"
485
            },
486
            "showPoints": "never",
487
            "spanNulls": false,
488
            "stacking": {
489
              "group": "A",
490
              "mode": "none"
491
            },
492
            "thresholdsStyle": {
493
              "mode": "off"
494
            }
495
          },
496
          "links": [],
497
          "mappings": [],
498
          "min": 0,
499
          "thresholds": {
500
            "mode": "absolute",
501
            "steps": [
502
              {
503
                "color": "green",
504
                "value": null
505
              },
506
              {
507
                "color": "red",
508
                "value": 80
509
              }
510
            ]
511
          },
512
          "unit": "short"
513
        },
514
        "overrides": [
515
          {
516
            "matcher": {
517
              "id": "byRegexp",
518
              "options": ".*error"
519
            },
520
            "properties": [
521
              {
522
                "id": "color",
523
                "value": {
524
                  "fixedColor": "red",
525
                  "mode": "fixed"
526
                }
527
              }
528
            ]
529
          },
530
          {
531
            "matcher": {
532
              "id": "byRegexp",
533
              "options": ".*requeue.*"
534
            },
535
            "properties": [
536
              {
537
                "id": "color",
538
                "value": {
539
                  "fixedColor": "blue",
540
                  "mode": "fixed"
541
                }
542
              }
543
            ]
544
          },
545
          {
546
            "matcher": {
547
              "id": "byRegexp",
548
              "options": ".*success"
549
            },
550
            "properties": [
551
              {
552
                "id": "color",
553
                "value": {
554
                  "fixedColor": "green",
555
                  "mode": "fixed"
556
                }
557
              }
558
            ]
559
          }
560
        ]
561
      },
562
      "gridPos": {
563
        "h": 9,
564
        "w": 8,
565
        "x": 8,
566
        "y": 10
567
      },
568
      "id": 75,
569
      "links": [],
570
      "options": {
571
        "legend": {
572
          "calcs": [
573
            "lastNotNull"
574
          ],
575
          "displayMode": "table",
576
          "placement": "bottom",
577
          "showLegend": true
578
        },
579
        "tooltip": {
580
          "mode": "single",
581
          "sort": "none"
582
        }
583
      },
584
      "pluginVersion": "9.3.6",
585
      "targets": [
586
        {
587
          "datasource": {
588
            "uid": "$datasource"
589
          },
590
          "editorMode": "code",
591
          "expr": "topk(20, sum(rate(controller_runtime_reconcile_total{job=\"ws-daemon\", pod=~\"$pod\", controller=~\"$controller\"}[5m])) by (cluster, pod, controller, result))",
592
          "format": "time_series",
593
          "intervalFactor": 2,
594
          "legendFormat": "{{cluster}} - {{pod}} - {{controller}} - {{result}}",
595
          "range": true,
596
          "refId": "A"
597
        }
598
      ],
599
      "title": "Reconciliations",
600
      "type": "timeseries"
601
    },
602
    {
603
      "datasource": {
604
        "type": "prometheus",
605
        "uid": "$datasource"
606
      },
607
      "fieldConfig": {
608
        "defaults": {
609
          "custom": {
610
            "hideFrom": {
611
              "legend": false,
612
              "tooltip": false,
613
              "viz": false
614
            },
615
            "scaleDistribution": {
616
              "type": "linear"
617
            }
618
          },
619
          "links": []
620
        },
621
        "overrides": []
622
      },
623
      "gridPos": {
624
        "h": 9,
625
        "w": 8,
626
        "x": 16,
627
        "y": 10
628
      },
629
      "id": 77,
630
      "links": [],
631
      "options": {
632
        "calculate": false,
633
        "cellGap": 1,
634
        "color": {
635
          "exponent": 0.5,
636
          "fill": "dark-orange",
637
          "mode": "scheme",
638
          "reverse": false,
639
          "scale": "exponential",
640
          "scheme": "Spectral",
641
          "steps": 64
642
        },
643
        "exemplars": {
644
          "color": "rgba(255,0,255,0.7)"
645
        },
646
        "filterValues": {
647
          "le": 1e-9
648
        },
649
        "legend": {
650
          "show": true
651
        },
652
        "rowsFrame": {
653
          "layout": "auto"
654
        },
655
        "tooltip": {
656
          "show": true,
657
          "yHistogram": false
658
        },
659
        "yAxis": {
660
          "axisPlacement": "left",
661
          "reverse": false,
662
          "unit": "s"
663
        }
664
      },
665
      "pluginVersion": "9.5.2",
666
      "targets": [
667
        {
668
          "datasource": {
669
            "uid": "$datasource"
670
          },
671
          "editorMode": "code",
672
          "expr": "sum(rate(workqueue_work_duration_seconds_bucket{job=\"ws-daemon\", pod=~\"$pod\", name=~\"$controller\"}[$__rate_interval])) by (le)",
673
          "format": "heatmap",
674
          "intervalFactor": 2,
675
          "legendFormat": "__auto",
676
          "range": true,
677
          "refId": "A"
678
        }
679
      ],
680
      "title": "Reconcile Work Duration",
681
      "type": "heatmap"
682
    },
683
    {
684
      "datasource": {
685
        "type": "prometheus",
686
        "uid": "$datasource"
687
      },
688
      "fieldConfig": {
689
        "defaults": {
690
          "color": {
691
            "mode": "palette-classic"
692
          },
693
          "custom": {
694
            "axisCenteredZero": false,
695
            "axisColorMode": "text",
696
            "axisLabel": "",
697
            "axisPlacement": "auto",
698
            "barAlignment": 0,
699
            "drawStyle": "line",
700
            "fillOpacity": 10,
701
            "gradientMode": "none",
702
            "hideFrom": {
703
              "legend": false,
704
              "tooltip": false,
705
              "viz": false
706
            },
707
            "lineInterpolation": "linear",
708
            "lineWidth": 1,
709
            "pointSize": 5,
710
            "scaleDistribution": {
711
              "type": "linear"
712
            },
713
            "showPoints": "never",
714
            "spanNulls": false,
715
            "stacking": {
716
              "group": "A",
717
              "mode": "none"
718
            },
719
            "thresholdsStyle": {
720
              "mode": "off"
721
            }
722
          },
723
          "links": [],
724
          "mappings": [],
725
          "min": 0,
726
          "thresholds": {
727
            "mode": "absolute",
728
            "steps": [
729
              {
730
                "color": "green",
731
                "value": null
732
              },
733
              {
734
                "color": "red",
735
                "value": 80
736
              }
737
            ]
738
          },
739
          "unit": "s"
740
        },
741
        "overrides": [
742
          {
743
            "matcher": {
744
              "id": "byRegexp",
745
              "options": ".*error"
746
            },
747
            "properties": [
748
              {
749
                "id": "color",
750
                "value": {
751
                  "fixedColor": "red",
752
                  "mode": "fixed"
753
                }
754
              }
755
            ]
756
          },
757
          {
758
            "matcher": {
759
              "id": "byRegexp",
760
              "options": ".*requeue.*"
761
            },
762
            "properties": [
763
              {
764
                "id": "color",
765
                "value": {
766
                  "fixedColor": "blue",
767
                  "mode": "fixed"
768
                }
769
              }
770
            ]
771
          },
772
          {
773
            "matcher": {
774
              "id": "byRegexp",
775
              "options": ".*success"
776
            },
777
            "properties": [
778
              {
779
                "id": "color",
780
                "value": {
781
                  "fixedColor": "green",
782
                  "mode": "fixed"
783
                }
784
              }
785
            ]
786
          }
787
        ]
788
      },
789
      "gridPos": {
790
        "h": 9,
791
        "w": 8,
792
        "x": 0,
793
        "y": 19
794
      },
795
      "id": 78,
796
      "links": [],
797
      "options": {
798
        "legend": {
799
          "calcs": [
800
            "lastNotNull"
801
          ],
802
          "displayMode": "table",
803
          "placement": "bottom",
804
          "showLegend": true
805
        },
806
        "tooltip": {
807
          "mode": "single",
808
          "sort": "none"
809
        }
810
      },
811
      "pluginVersion": "9.3.6",
812
      "targets": [
813
        {
814
          "datasource": {
815
            "type": "prometheus",
816
            "uid": "$datasource"
817
          },
818
          "editorMode": "code",
819
          "expr": "topk(20, max(workqueue_longest_running_processor_seconds{job=\"ws-daemon\", pod=~\"$pod\", name=~\"$controller\"}) by (cluster, pod, name))",
820
          "hide": false,
821
          "legendFormat": "{{cluster}} - {{pod}} - {{name}}",
822
          "range": true,
823
          "refId": "B"
824
        }
825
      ],
826
      "title": "Longest running reconciliation",
827
      "type": "timeseries"
828
    },
829
    {
830
      "collapsed": true,
831
      "datasource": {
832
        "type": "datasource",
833
        "uid": "grafana"
834
      },
835
      "gridPos": {
836
        "h": 1,
837
        "w": 24,
838
        "x": 0,
839
        "y": 28
840
      },
841
      "id": 52,
842
      "panels": [
843
        {
844
          "aliasColors": {},
845
          "bars": false,
846
          "dashLength": 10,
847
          "dashes": false,
848
          "datasource": {
849
            "uid": "$datasource"
850
          },
851
          "description": "",
852
          "fieldConfig": {
853
            "defaults": {
854
              "custom": {},
855
              "links": []
856
            },
857
            "overrides": []
858
          },
859
          "fill": 1,
860
          "fillGradient": 0,
861
          "gridPos": {
862
            "h": 9,
863
            "w": 12,
864
            "x": 0,
865
            "y": 57
866
          },
867
          "hiddenSeries": false,
868
          "id": 54,
869
          "legend": {
870
            "alignAsTable": true,
871
            "avg": false,
872
            "current": true,
873
            "max": true,
874
            "min": false,
875
            "show": true,
876
            "total": false,
877
            "values": true
878
          },
879
          "lines": true,
880
          "linewidth": 1,
881
          "links": [],
882
          "nullPointMode": "null",
883
          "options": {
884
            "alertThreshold": true
885
          },
886
          "percentage": false,
887
          "pluginVersion": "7.4.3",
888
          "pointradius": 2,
889
          "points": false,
890
          "renderer": "flot",
891
          "repeatDirection": "h",
892
          "seriesOverrides": [],
893
          "spaceLength": 10,
894
          "stack": false,
895
          "steppedLine": false,
896
          "targets": [
897
            {
898
              "datasource": {
899
                "uid": "$datasource"
900
              },
901
              "expr": "sum(rate(grpc_server_handled_total{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[1m])) by (grpc_code, grpc_method, cluster)",
902
              "interval": "",
903
              "legendFormat": "{{cluster}} - {{grpc_method}} - {{grpc_code}}",
904
              "queryType": "randomWalk",
905
              "refId": "A"
906
            }
907
          ],
908
          "thresholds": [],
909
          "timeRegions": [],
910
          "title": "gRPC calls handled (Server-side)",
911
          "tooltip": {
912
            "shared": true,
913
            "sort": 2,
914
            "value_type": "individual"
915
          },
916
          "type": "graph",
917
          "xaxis": {
918
            "mode": "time",
919
            "show": true,
920
            "values": []
921
          },
922
          "yaxes": [
923
            {
924
              "decimals": 2,
925
              "format": "reqps",
926
              "logBase": 1,
927
              "min": "0",
928
              "show": true
929
            },
930
            {
931
              "format": "reqps",
932
              "logBase": 1,
933
              "show": true
934
            }
935
          ],
936
          "yaxis": {
937
            "align": false
938
          }
939
        },
940
        {
941
          "aliasColors": {},
942
          "bars": false,
943
          "dashLength": 10,
944
          "dashes": false,
945
          "datasource": {
946
            "uid": "$datasource"
947
          },
948
          "description": "",
949
          "fieldConfig": {
950
            "defaults": {
951
              "custom": {},
952
              "links": []
953
            },
954
            "overrides": []
955
          },
956
          "fill": 1,
957
          "fillGradient": 0,
958
          "gridPos": {
959
            "h": 9,
960
            "w": 12,
961
            "x": 12,
962
            "y": 57
963
          },
964
          "hiddenSeries": false,
965
          "id": 56,
966
          "legend": {
967
            "alignAsTable": true,
968
            "avg": false,
969
            "current": true,
970
            "max": true,
971
            "min": false,
972
            "show": true,
973
            "total": false,
974
            "values": true
975
          },
976
          "lines": true,
977
          "linewidth": 1,
978
          "links": [
979
            {
980
              "targetBlank": true,
981
              "title": "How avg help us configure representative Prometheus histogram buckets",
982
              "url": "https://youtu.be/zk77VS98Em8?t=908"
983
            }
984
          ],
985
          "nullPointMode": "null",
986
          "options": {
987
            "alertThreshold": true
988
          },
989
          "percentage": false,
990
          "pluginVersion": "7.4.3",
991
          "pointradius": 2,
992
          "points": false,
993
          "renderer": "flot",
994
          "repeatDirection": "h",
995
          "seriesOverrides": [],
996
          "spaceLength": 10,
997
          "stack": false,
998
          "steppedLine": false,
999
          "targets": [
1000
            {
1001
              "datasource": {
1002
                "uid": "$datasource"
1003
              },
1004
              "expr": "sum(rate(grpc_server_started_total{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[1m])) by (grpc_code, grpc_method, cluster)",
1005
              "interval": "",
1006
              "legendFormat": "{{cluster}} - {{grpc_method}} - Started",
1007
              "queryType": "randomWalk",
1008
              "refId": "A"
1009
            },
1010
            {
1011
              "datasource": {
1012
                "uid": "$datasource"
1013
              },
1014
              "expr": "sum(rate(grpc_server_handled_total{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[1m])) by (grpc_method, cluster)",
1015
              "hide": false,
1016
              "interval": "",
1017
              "legendFormat": "{{cluster}} - {{grpc_method}} - Finished",
1018
              "queryType": "randomWalk",
1019
              "refId": "B"
1020
            }
1021
          ],
1022
          "thresholds": [],
1023
          "timeRegions": [],
1024
          "title": "gRPC start and finish rate",
1025
          "tooltip": {
1026
            "shared": true,
1027
            "sort": 2,
1028
            "value_type": "individual"
1029
          },
1030
          "type": "graph",
1031
          "xaxis": {
1032
            "mode": "time",
1033
            "show": true,
1034
            "values": []
1035
          },
1036
          "yaxes": [
1037
            {
1038
              "decimals": 2,
1039
              "format": "reqps",
1040
              "logBase": 1,
1041
              "min": "0",
1042
              "show": true
1043
            },
1044
            {
1045
              "format": "reqps",
1046
              "logBase": 1,
1047
              "show": true
1048
            }
1049
          ],
1050
          "yaxis": {
1051
            "align": false
1052
          }
1053
        },
1054
        {
1055
          "aliasColors": {},
1056
          "bars": false,
1057
          "dashLength": 10,
1058
          "dashes": false,
1059
          "datasource": {
1060
            "uid": "$datasource"
1061
          },
1062
          "description": "",
1063
          "fieldConfig": {
1064
            "defaults": {
1065
              "custom": {},
1066
              "links": []
1067
            },
1068
            "overrides": []
1069
          },
1070
          "fill": 1,
1071
          "fillGradient": 0,
1072
          "gridPos": {
1073
            "h": 9,
1074
            "w": 12,
1075
            "x": 0,
1076
            "y": 66
1077
          },
1078
          "hiddenSeries": false,
1079
          "id": 58,
1080
          "legend": {
1081
            "alignAsTable": true,
1082
            "avg": false,
1083
            "current": true,
1084
            "max": true,
1085
            "min": false,
1086
            "show": true,
1087
            "total": false,
1088
            "values": true
1089
          },
1090
          "lines": true,
1091
          "linewidth": 1,
1092
          "links": [],
1093
          "maxPerRow": 2,
1094
          "nullPointMode": "null",
1095
          "options": {
1096
            "alertThreshold": true
1097
          },
1098
          "percentage": false,
1099
          "pluginVersion": "7.4.3",
1100
          "pointradius": 2,
1101
          "points": false,
1102
          "renderer": "flot",
1103
          "repeat": "grpc_method",
1104
          "repeatDirection": "h",
1105
          "seriesOverrides": [],
1106
          "spaceLength": 10,
1107
          "stack": false,
1108
          "steppedLine": false,
1109
          "targets": [
1110
            {
1111
              "datasource": {
1112
                "uid": "$datasource"
1113
              },
1114
              "expr": "histogram_quantile(0.99, \n  sum(\n    rate(grpc_server_handling_seconds_bucket{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method, le)\n)",
1115
              "interval": "",
1116
              "legendFormat": "{{cluster}} - {{grpc_method}} - 99th Percentile",
1117
              "queryType": "randomWalk",
1118
              "refId": "A"
1119
            },
1120
            {
1121
              "datasource": {
1122
                "uid": "$datasource"
1123
              },
1124
              "expr": "histogram_quantile(0.95, \n  sum(\n    rate(grpc_server_handling_seconds_bucket{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method, le)\n)",
1125
              "interval": "",
1126
              "legendFormat": "{{cluster}} - {{grpc_method}} - 95th Percentile",
1127
              "queryType": "randomWalk",
1128
              "refId": "B"
1129
            },
1130
            {
1131
              "datasource": {
1132
                "uid": "$datasource"
1133
              },
1134
              "expr": "histogram_quantile(0.50, \n  sum(\n    rate(grpc_server_handling_seconds_bucket{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method, le)\n)",
1135
              "interval": "",
1136
              "legendFormat": "{{cluster}} - {{grpc_method}} - 50th Percentile",
1137
              "queryType": "randomWalk",
1138
              "refId": "C"
1139
            },
1140
            {
1141
              "datasource": {
1142
                "uid": "$datasource"
1143
              },
1144
              "expr": "  sum(\n    rate(grpc_server_handling_seconds_sum{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method)\n  /\n  sum(\n    rate(grpc_server_handling_seconds_count{job=\"ws-daemon\", cluster=~\"$cluster\", grpc_method=\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method)",
1145
              "hide": false,
1146
              "interval": "",
1147
              "legendFormat": "{{cluster}} - {{grpc_method}} - avg",
1148
              "queryType": "randomWalk",
1149
              "refId": "D"
1150
            }
1151
          ],
1152
          "thresholds": [],
1153
          "timeRegions": [],
1154
          "title": "$grpc_method: Time handling gRPC calls",
1155
          "tooltip": {
1156
            "shared": true,
1157
            "sort": 0,
1158
            "value_type": "individual"
1159
          },
1160
          "type": "graph",
1161
          "xaxis": {
1162
            "mode": "time",
1163
            "show": true,
1164
            "values": []
1165
          },
1166
          "yaxes": [
1167
            {
1168
              "format": "s",
1169
              "logBase": 1,
1170
              "min": "0",
1171
              "show": true
1172
            },
1173
            {
1174
              "format": "reqps",
1175
              "logBase": 1,
1176
              "show": true
1177
            }
1178
          ],
1179
          "yaxis": {
1180
            "align": false
1181
          }
1182
        }
1183
      ],
1184
      "targets": [
1185
        {
1186
          "datasource": {
1187
            "type": "datasource",
1188
            "uid": "grafana"
1189
          },
1190
          "refId": "A"
1191
        }
1192
      ],
1193
      "title": "gRPC Metrics",
1194
      "type": "row"
1195
    },
1196
    {
1197
      "collapsed": false,
1198
      "datasource": {
1199
        "type": "datasource",
1200
        "uid": "grafana"
1201
      },
1202
      "gridPos": {
1203
        "h": 1,
1204
        "w": 24,
1205
        "x": 0,
1206
        "y": 29
1207
      },
1208
      "id": 16,
1209
      "panels": [],
1210
      "targets": [
1211
        {
1212
          "datasource": {
1213
            "type": "datasource",
1214
            "uid": "grafana"
1215
          },
1216
          "refId": "A"
1217
        }
1218
      ],
1219
      "title": "Pod Metrics",
1220
      "type": "row"
1221
    },
1222
    {
1223
      "datasource": {
1224
        "uid": "$datasource"
1225
      },
1226
      "fieldConfig": {
1227
        "defaults": {
1228
          "color": {
1229
            "mode": "palette-classic"
1230
          },
1231
          "custom": {
1232
            "axisCenteredZero": false,
1233
            "axisColorMode": "text",
1234
            "axisLabel": "",
1235
            "axisPlacement": "auto",
1236
            "barAlignment": 0,
1237
            "drawStyle": "line",
1238
            "fillOpacity": 0,
1239
            "gradientMode": "none",
1240
            "hideFrom": {
1241
              "legend": false,
1242
              "tooltip": false,
1243
              "viz": false
1244
            },
1245
            "lineInterpolation": "linear",
1246
            "lineWidth": 1,
1247
            "pointSize": 5,
1248
            "scaleDistribution": {
1249
              "type": "linear"
1250
            },
1251
            "showPoints": "auto",
1252
            "spanNulls": false,
1253
            "stacking": {
1254
              "group": "A",
1255
              "mode": "none"
1256
            },
1257
            "thresholdsStyle": {
1258
              "mode": "off"
1259
            }
1260
          },
1261
          "mappings": [],
1262
          "thresholds": {
1263
            "mode": "absolute",
1264
            "steps": [
1265
              {
1266
                "color": "green",
1267
                "value": null
1268
              },
1269
              {
1270
                "color": "red",
1271
                "value": 80
1272
              }
1273
            ]
1274
          }
1275
        },
1276
        "overrides": []
1277
      },
1278
      "gridPos": {
1279
        "h": 7,
1280
        "w": 10,
1281
        "x": 0,
1282
        "y": 30
1283
      },
1284
      "id": 63,
1285
      "options": {
1286
        "legend": {
1287
          "calcs": [],
1288
          "displayMode": "table",
1289
          "placement": "right",
1290
          "showLegend": true
1291
        },
1292
        "tooltip": {
1293
          "mode": "single",
1294
          "sort": "none"
1295
        }
1296
      },
1297
      "targets": [
1298
        {
1299
          "datasource": {
1300
            "uid": "$datasource"
1301
          },
1302
          "exemplar": true,
1303
          "expr": "kube_pod_container_info{cluster=~\"$cluster\", pod=~\"$pod\", image=~\".+\", container=\"ws-daemon\"}",
1304
          "interval": "",
1305
          "legendFormat": "{{cluster}} - {{image}}",
1306
          "queryType": "randomWalk",
1307
          "refId": "A"
1308
        }
1309
      ],
1310
      "title": "Container image version",
1311
      "type": "timeseries"
1312
    },
1313
    {
1314
      "aliasColors": {},
1315
      "bars": false,
1316
      "dashLength": 10,
1317
      "dashes": false,
1318
      "datasource": {
1319
        "uid": "$datasource"
1320
      },
1321
      "decimals": 2,
1322
      "fill": 1,
1323
      "fillGradient": 0,
1324
      "gridPos": {
1325
        "h": 7,
1326
        "w": 7,
1327
        "x": 10,
1328
        "y": 30
1329
      },
1330
      "hiddenSeries": false,
1331
      "id": 2,
1332
      "legend": {
1333
        "alignAsTable": true,
1334
        "avg": true,
1335
        "current": true,
1336
        "max": true,
1337
        "min": false,
1338
        "show": true,
1339
        "sort": "current",
1340
        "sortDesc": true,
1341
        "total": false,
1342
        "values": true
1343
      },
1344
      "lines": true,
1345
      "linewidth": 1,
1346
      "nullPointMode": "null",
1347
      "options": {
1348
        "alertThreshold": true
1349
      },
1350
      "percentage": false,
1351
      "pluginVersion": "9.5.2",
1352
      "pointradius": 2,
1353
      "points": false,
1354
      "renderer": "flot",
1355
      "seriesOverrides": [],
1356
      "spaceLength": 10,
1357
      "stack": false,
1358
      "steppedLine": false,
1359
      "targets": [
1360
        {
1361
          "datasource": {
1362
            "uid": "$datasource"
1363
          },
1364
          "expr": "sum(\n  rate(container_cpu_usage_seconds_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1365
          "interval": "",
1366
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Cores being used",
1367
          "queryType": "randomWalk",
1368
          "refId": "A"
1369
        }
1370
      ],
1371
      "thresholds": [],
1372
      "timeRegions": [],
1373
      "title": "CPU Utilization",
1374
      "tooltip": {
1375
        "shared": true,
1376
        "sort": 2,
1377
        "value_type": "individual"
1378
      },
1379
      "type": "graph",
1380
      "xaxis": {
1381
        "mode": "time",
1382
        "show": true,
1383
        "values": []
1384
      },
1385
      "yaxes": [
1386
        {
1387
          "$$hashKey": "object:564",
1388
          "decimals": 2,
1389
          "format": "short",
1390
          "logBase": 1,
1391
          "min": "0",
1392
          "show": true
1393
        },
1394
        {
1395
          "$$hashKey": "object:565",
1396
          "format": "short",
1397
          "logBase": 1,
1398
          "show": true
1399
        }
1400
      ],
1401
      "yaxis": {
1402
        "align": false
1403
      }
1404
    },
1405
    {
1406
      "aliasColors": {},
1407
      "bars": false,
1408
      "dashLength": 10,
1409
      "dashes": false,
1410
      "datasource": {
1411
        "uid": "$datasource"
1412
      },
1413
      "description": "Saturation > 100% means that the container is requesting more than its limits.\n\nKubernetes will start to throttle CPU when that happens. That's a sign of degraded performance.\n\n'No Data' indicates that the pod has no CPU limits.",
1414
      "fill": 1,
1415
      "fillGradient": 0,
1416
      "gridPos": {
1417
        "h": 7,
1418
        "w": 7,
1419
        "x": 17,
1420
        "y": 30
1421
      },
1422
      "hiddenSeries": false,
1423
      "id": 4,
1424
      "legend": {
1425
        "avg": false,
1426
        "current": false,
1427
        "max": false,
1428
        "min": false,
1429
        "show": true,
1430
        "total": false,
1431
        "values": false
1432
      },
1433
      "lines": true,
1434
      "linewidth": 1,
1435
      "nullPointMode": "null",
1436
      "options": {
1437
        "alertThreshold": true
1438
      },
1439
      "percentage": false,
1440
      "pluginVersion": "9.5.2",
1441
      "pointradius": 2,
1442
      "points": false,
1443
      "renderer": "flot",
1444
      "seriesOverrides": [
1445
        {
1446
          "alias": "/.*CPU Throttles/",
1447
          "yaxis": 2
1448
        }
1449
      ],
1450
      "spaceLength": 10,
1451
      "stack": false,
1452
      "steppedLine": false,
1453
      "targets": [
1454
        {
1455
          "datasource": {
1456
            "uid": "$datasource"
1457
          },
1458
          "expr": "sum(\n  rate(container_cpu_usage_seconds_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)\n/\nsum(\n  kube_pod_container_resource_limits_cpu_cores{container!=\"POD\", cluster=\"$cluster\", pod=~\"$pod\"}\n) by (pod, cluster, node)\n",
1459
          "interval": "",
1460
          "legendFormat": "{{cluster}} - {{node} - {{pod}} - CPU Saturation",
1461
          "queryType": "randomWalk",
1462
          "refId": "A"
1463
        },
1464
        {
1465
          "datasource": {
1466
            "uid": "$datasource"
1467
          },
1468
          "expr": "sum(\nrate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod)",
1469
          "interval": "",
1470
          "legendFormat": "{{pod}} - CPU Throttles",
1471
          "queryType": "randomWalk",
1472
          "refId": "B"
1473
        }
1474
      ],
1475
      "thresholds": [],
1476
      "timeRegions": [],
1477
      "title": "CPU Saturation",
1478
      "tooltip": {
1479
        "shared": true,
1480
        "sort": 2,
1481
        "value_type": "individual"
1482
      },
1483
      "type": "graph",
1484
      "xaxis": {
1485
        "mode": "time",
1486
        "show": true,
1487
        "values": []
1488
      },
1489
      "yaxes": [
1490
        {
1491
          "decimals": 2,
1492
          "format": "percentunit",
1493
          "logBase": 1,
1494
          "min": "0",
1495
          "show": true
1496
        },
1497
        {
1498
          "format": "s",
1499
          "logBase": 1,
1500
          "show": true
1501
        }
1502
      ],
1503
      "yaxis": {
1504
        "align": false
1505
      }
1506
    },
1507
    {
1508
      "aliasColors": {},
1509
      "bars": false,
1510
      "dashLength": 10,
1511
      "dashes": false,
1512
      "datasource": {
1513
        "uid": "$datasource"
1514
      },
1515
      "fill": 1,
1516
      "fillGradient": 0,
1517
      "gridPos": {
1518
        "h": 7,
1519
        "w": 12,
1520
        "x": 0,
1521
        "y": 37
1522
      },
1523
      "hiddenSeries": false,
1524
      "id": 6,
1525
      "legend": {
1526
        "alignAsTable": true,
1527
        "avg": true,
1528
        "current": true,
1529
        "max": true,
1530
        "min": false,
1531
        "show": true,
1532
        "total": false,
1533
        "values": true
1534
      },
1535
      "lines": true,
1536
      "linewidth": 1,
1537
      "nullPointMode": "null",
1538
      "options": {
1539
        "alertThreshold": true
1540
      },
1541
      "percentage": false,
1542
      "pluginVersion": "9.5.2",
1543
      "pointradius": 2,
1544
      "points": false,
1545
      "renderer": "flot",
1546
      "seriesOverrides": [],
1547
      "spaceLength": 10,
1548
      "stack": false,
1549
      "steppedLine": false,
1550
      "targets": [
1551
        {
1552
          "datasource": {
1553
            "uid": "$datasource"
1554
          },
1555
          "expr": "sum(container_memory_working_set_bytes{container!=\"POD\", container!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}) by (pod, cluster, node)",
1556
          "interval": "",
1557
          "legendFormat": "{{cluster}} - {{node}} - {{pod}}",
1558
          "queryType": "randomWalk",
1559
          "refId": "A"
1560
        }
1561
      ],
1562
      "thresholds": [],
1563
      "timeRegions": [],
1564
      "title": "Memory Utilization",
1565
      "tooltip": {
1566
        "shared": true,
1567
        "sort": 2,
1568
        "value_type": "individual"
1569
      },
1570
      "type": "graph",
1571
      "xaxis": {
1572
        "mode": "time",
1573
        "show": true,
1574
        "values": []
1575
      },
1576
      "yaxes": [
1577
        {
1578
          "format": "bytes",
1579
          "logBase": 1,
1580
          "min": "0",
1581
          "show": true
1582
        },
1583
        {
1584
          "format": "short",
1585
          "logBase": 1,
1586
          "show": true
1587
        }
1588
      ],
1589
      "yaxis": {
1590
        "align": false
1591
      }
1592
    },
1593
    {
1594
      "aliasColors": {},
1595
      "bars": false,
1596
      "dashLength": 10,
1597
      "dashes": false,
1598
      "datasource": {
1599
        "uid": "$datasource"
1600
      },
1601
      "decimals": 4,
1602
      "description": "Memory can't be throttled. When a container reaches 100% of its memory limits,  Kubernetes will kill the container and restart it.\n\n'No Data' indicates that the pod doesn't have Memory limits.",
1603
      "fill": 1,
1604
      "fillGradient": 0,
1605
      "gridPos": {
1606
        "h": 7,
1607
        "w": 12,
1608
        "x": 12,
1609
        "y": 37
1610
      },
1611
      "hiddenSeries": false,
1612
      "id": 8,
1613
      "legend": {
1614
        "alignAsTable": true,
1615
        "avg": true,
1616
        "current": true,
1617
        "max": true,
1618
        "min": false,
1619
        "show": true,
1620
        "total": false,
1621
        "values": true
1622
      },
1623
      "lines": true,
1624
      "linewidth": 1,
1625
      "nullPointMode": "null",
1626
      "options": {
1627
        "alertThreshold": true
1628
      },
1629
      "percentage": false,
1630
      "pluginVersion": "9.5.2",
1631
      "pointradius": 2,
1632
      "points": false,
1633
      "renderer": "flot",
1634
      "seriesOverrides": [],
1635
      "spaceLength": 10,
1636
      "stack": false,
1637
      "steppedLine": false,
1638
      "targets": [
1639
        {
1640
          "datasource": {
1641
            "uid": "$datasource"
1642
          },
1643
          "expr": "sum(\nrate(container_memory_working_set_bytes{container!=\"POD\", container!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)\n/\nsum(\n  kube_pod_container_resource_limits{container!=\"POD\", cluster=\"$cluster\", pod=~\"$pod\", resource=\"memory\"}\n) by (pod, cluster, node)\n",
1644
          "interval": "",
1645
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Memory Saturation",
1646
          "queryType": "randomWalk",
1647
          "refId": "A"
1648
        }
1649
      ],
1650
      "thresholds": [],
1651
      "timeRegions": [],
1652
      "title": "Memory Saturation",
1653
      "tooltip": {
1654
        "shared": true,
1655
        "sort": 2,
1656
        "value_type": "individual"
1657
      },
1658
      "type": "graph",
1659
      "xaxis": {
1660
        "mode": "time",
1661
        "show": true,
1662
        "values": []
1663
      },
1664
      "yaxes": [
1665
        {
1666
          "decimals": 2,
1667
          "format": "percentunit",
1668
          "logBase": 1,
1669
          "min": "0",
1670
          "show": true
1671
        },
1672
        {
1673
          "format": "short",
1674
          "logBase": 1,
1675
          "show": true
1676
        }
1677
      ],
1678
      "yaxis": {
1679
        "align": false
1680
      }
1681
    },
1682
    {
1683
      "aliasColors": {},
1684
      "bars": false,
1685
      "dashLength": 10,
1686
      "dashes": false,
1687
      "datasource": {
1688
        "uid": "$datasource"
1689
      },
1690
      "fill": 1,
1691
      "fillGradient": 0,
1692
      "gridPos": {
1693
        "h": 7,
1694
        "w": 8,
1695
        "x": 0,
1696
        "y": 44
1697
      },
1698
      "hiddenSeries": false,
1699
      "id": 10,
1700
      "legend": {
1701
        "alignAsTable": true,
1702
        "avg": false,
1703
        "current": true,
1704
        "max": false,
1705
        "min": false,
1706
        "show": true,
1707
        "total": false,
1708
        "values": true
1709
      },
1710
      "lines": true,
1711
      "linewidth": 1,
1712
      "nullPointMode": "null",
1713
      "options": {
1714
        "alertThreshold": true
1715
      },
1716
      "percentage": false,
1717
      "pluginVersion": "9.5.2",
1718
      "pointradius": 2,
1719
      "points": false,
1720
      "renderer": "flot",
1721
      "seriesOverrides": [],
1722
      "spaceLength": 10,
1723
      "stack": false,
1724
      "steppedLine": false,
1725
      "targets": [
1726
        {
1727
          "datasource": {
1728
            "uid": "$datasource"
1729
          },
1730
          "expr": "sum (\n  rate(container_network_receive_bytes_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1731
          "interval": "",
1732
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Received",
1733
          "queryType": "randomWalk",
1734
          "refId": "A"
1735
        },
1736
        {
1737
          "datasource": {
1738
            "uid": "$datasource"
1739
          },
1740
          "expr": "sum (\n  rate(container_network_transmit_bytes_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1741
          "interval": "",
1742
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Transmitted",
1743
          "queryType": "randomWalk",
1744
          "refId": "B"
1745
        }
1746
      ],
1747
      "thresholds": [],
1748
      "timeRegions": [],
1749
      "title": "Network Utilization",
1750
      "tooltip": {
1751
        "shared": true,
1752
        "sort": 2,
1753
        "value_type": "individual"
1754
      },
1755
      "type": "graph",
1756
      "xaxis": {
1757
        "mode": "time",
1758
        "show": true,
1759
        "values": []
1760
      },
1761
      "yaxes": [
1762
        {
1763
          "format": "binBps",
1764
          "logBase": 1,
1765
          "min": "0",
1766
          "show": true
1767
        },
1768
        {
1769
          "format": "short",
1770
          "logBase": 1,
1771
          "show": true
1772
        }
1773
      ],
1774
      "yaxis": {
1775
        "align": false
1776
      }
1777
    },
1778
    {
1779
      "aliasColors": {},
1780
      "bars": false,
1781
      "dashLength": 10,
1782
      "dashes": false,
1783
      "datasource": {
1784
        "uid": "$datasource"
1785
      },
1786
      "fill": 1,
1787
      "fillGradient": 0,
1788
      "gridPos": {
1789
        "h": 7,
1790
        "w": 8,
1791
        "x": 8,
1792
        "y": 44
1793
      },
1794
      "hiddenSeries": false,
1795
      "id": 44,
1796
      "legend": {
1797
        "alignAsTable": true,
1798
        "avg": false,
1799
        "current": true,
1800
        "max": false,
1801
        "min": false,
1802
        "rightSide": false,
1803
        "show": true,
1804
        "total": false,
1805
        "values": true
1806
      },
1807
      "lines": true,
1808
      "linewidth": 1,
1809
      "nullPointMode": "null",
1810
      "options": {
1811
        "alertThreshold": true
1812
      },
1813
      "percentage": false,
1814
      "pluginVersion": "9.5.2",
1815
      "pointradius": 2,
1816
      "points": false,
1817
      "renderer": "flot",
1818
      "seriesOverrides": [],
1819
      "spaceLength": 10,
1820
      "stack": false,
1821
      "steppedLine": false,
1822
      "targets": [
1823
        {
1824
          "datasource": {
1825
            "uid": "$datasource"
1826
          },
1827
          "expr": "sum (\n  rate(container_network_receive_packets_dropped_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1828
          "interval": "",
1829
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Receive",
1830
          "queryType": "randomWalk",
1831
          "refId": "A"
1832
        },
1833
        {
1834
          "datasource": {
1835
            "uid": "$datasource"
1836
          },
1837
          "expr": "sum (\n  rate(container_network_transmit_packets_dropped_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1838
          "interval": "",
1839
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Transmit",
1840
          "queryType": "randomWalk",
1841
          "refId": "B"
1842
        }
1843
      ],
1844
      "thresholds": [],
1845
      "timeRegions": [],
1846
      "title": "Network Saturation (Packets Dropped)",
1847
      "tooltip": {
1848
        "shared": true,
1849
        "sort": 2,
1850
        "value_type": "individual"
1851
      },
1852
      "type": "graph",
1853
      "xaxis": {
1854
        "mode": "time",
1855
        "show": true,
1856
        "values": []
1857
      },
1858
      "yaxes": [
1859
        {
1860
          "decimals": 2,
1861
          "format": "pps",
1862
          "logBase": 1,
1863
          "min": "0",
1864
          "show": true
1865
        },
1866
        {
1867
          "format": "short",
1868
          "logBase": 1,
1869
          "show": true
1870
        }
1871
      ],
1872
      "yaxis": {
1873
        "align": false
1874
      }
1875
    },
1876
    {
1877
      "aliasColors": {},
1878
      "bars": false,
1879
      "dashLength": 10,
1880
      "dashes": false,
1881
      "datasource": {
1882
        "uid": "$datasource"
1883
      },
1884
      "fill": 1,
1885
      "fillGradient": 0,
1886
      "gridPos": {
1887
        "h": 7,
1888
        "w": 8,
1889
        "x": 16,
1890
        "y": 44
1891
      },
1892
      "hiddenSeries": false,
1893
      "id": 46,
1894
      "legend": {
1895
        "alignAsTable": true,
1896
        "avg": false,
1897
        "current": true,
1898
        "max": false,
1899
        "min": false,
1900
        "show": true,
1901
        "total": false,
1902
        "values": true
1903
      },
1904
      "lines": true,
1905
      "linewidth": 1,
1906
      "nullPointMode": "null",
1907
      "options": {
1908
        "alertThreshold": true
1909
      },
1910
      "percentage": false,
1911
      "pluginVersion": "9.5.2",
1912
      "pointradius": 2,
1913
      "points": false,
1914
      "renderer": "flot",
1915
      "seriesOverrides": [],
1916
      "spaceLength": 10,
1917
      "stack": false,
1918
      "steppedLine": false,
1919
      "targets": [
1920
        {
1921
          "datasource": {
1922
            "uid": "$datasource"
1923
          },
1924
          "expr": "sum (\n  rate(container_network_receive_errors_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1925
          "interval": "",
1926
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Received",
1927
          "queryType": "randomWalk",
1928
          "refId": "A"
1929
        },
1930
        {
1931
          "datasource": {
1932
            "uid": "$datasource"
1933
          },
1934
          "expr": "sum (\n  rate(container_network_transmit_errors_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
1935
          "interval": "",
1936
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Transmitted",
1937
          "queryType": "randomWalk",
1938
          "refId": "B"
1939
        }
1940
      ],
1941
      "thresholds": [],
1942
      "timeRegions": [],
1943
      "title": "Network Errors",
1944
      "tooltip": {
1945
        "shared": true,
1946
        "sort": 2,
1947
        "value_type": "individual"
1948
      },
1949
      "type": "graph",
1950
      "xaxis": {
1951
        "mode": "time",
1952
        "show": true,
1953
        "values": []
1954
      },
1955
      "yaxes": [
1956
        {
1957
          "decimals": 2,
1958
          "format": "Errors/s",
1959
          "logBase": 1,
1960
          "min": "0",
1961
          "show": true
1962
        },
1963
        {
1964
          "format": "short",
1965
          "logBase": 1,
1966
          "show": true
1967
        }
1968
      ],
1969
      "yaxis": {
1970
        "align": false
1971
      }
1972
    },
1973
    {
1974
      "aliasColors": {},
1975
      "bars": false,
1976
      "dashLength": 10,
1977
      "dashes": false,
1978
      "datasource": {
1979
        "uid": "$datasource"
1980
      },
1981
      "decimals": 4,
1982
      "description": "",
1983
      "fill": 1,
1984
      "fillGradient": 0,
1985
      "gridPos": {
1986
        "h": 9,
1987
        "w": 8,
1988
        "x": 0,
1989
        "y": 51
1990
      },
1991
      "hiddenSeries": false,
1992
      "id": 36,
1993
      "legend": {
1994
        "alignAsTable": true,
1995
        "avg": true,
1996
        "current": true,
1997
        "max": true,
1998
        "min": false,
1999
        "show": true,
2000
        "total": false,
2001
        "values": true
2002
      },
2003
      "lines": true,
2004
      "linewidth": 1,
2005
      "nullPointMode": "null",
2006
      "options": {
2007
        "alertThreshold": true
2008
      },
2009
      "percentage": false,
2010
      "pluginVersion": "9.4.3",
2011
      "pointradius": 2,
2012
      "points": false,
2013
      "renderer": "flot",
2014
      "seriesOverrides": [],
2015
      "spaceLength": 10,
2016
      "stack": false,
2017
      "steppedLine": false,
2018
      "targets": [
2019
        {
2020
          "datasource": {
2021
            "uid": "$datasource"
2022
          },
2023
          "expr": "rate(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", pod=~\"$pod\"}[1m])",
2024
          "interval": "",
2025
          "legendFormat": "{{cluster}} - {{kubernetes_pod_node_name}} - {{pod}} ",
2026
          "queryType": "randomWalk",
2027
          "refId": "A"
2028
        }
2029
      ],
2030
      "thresholds": [],
2031
      "timeRegions": [],
2032
      "title": "Pod Restarts",
2033
      "tooltip": {
2034
        "shared": true,
2035
        "sort": 2,
2036
        "value_type": "individual"
2037
      },
2038
      "type": "graph",
2039
      "xaxis": {
2040
        "mode": "time",
2041
        "show": true,
2042
        "values": []
2043
      },
2044
      "yaxes": [
2045
        {
2046
          "decimals": 2,
2047
          "format": "short",
2048
          "logBase": 1,
2049
          "min": "0",
2050
          "show": true
2051
        },
2052
        {
2053
          "format": "short",
2054
          "logBase": 1,
2055
          "show": true
2056
        }
2057
      ],
2058
      "yaxis": {
2059
        "align": false
2060
      }
2061
    },
2062
    {
2063
      "aliasColors": {},
2064
      "bars": false,
2065
      "dashLength": 10,
2066
      "dashes": false,
2067
      "datasource": {
2068
        "uid": "$datasource"
2069
      },
2070
      "decimals": 0,
2071
      "description": "",
2072
      "fill": 1,
2073
      "fillGradient": 0,
2074
      "gridPos": {
2075
        "h": 9,
2076
        "w": 8,
2077
        "x": 8,
2078
        "y": 51
2079
      },
2080
      "hiddenSeries": false,
2081
      "id": 42,
2082
      "legend": {
2083
        "alignAsTable": true,
2084
        "avg": false,
2085
        "current": true,
2086
        "max": false,
2087
        "min": false,
2088
        "show": true,
2089
        "total": false,
2090
        "values": true
2091
      },
2092
      "lines": true,
2093
      "linewidth": 1,
2094
      "nullPointMode": "null",
2095
      "options": {
2096
        "alertThreshold": true
2097
      },
2098
      "percentage": false,
2099
      "pluginVersion": "9.4.3",
2100
      "pointradius": 2,
2101
      "points": false,
2102
      "renderer": "flot",
2103
      "seriesOverrides": [],
2104
      "spaceLength": 10,
2105
      "stack": false,
2106
      "steppedLine": false,
2107
      "targets": [
2108
        {
2109
          "datasource": {
2110
            "uid": "$datasource"
2111
          },
2112
          "expr": "kube_pod_container_status_running{cluster=~\"$cluster\", pod=~\"$pod\"} == 1 ",
2113
          "interval": "",
2114
          "legendFormat": "{{pod}}  - RUNNING",
2115
          "queryType": "randomWalk",
2116
          "refId": "A"
2117
        },
2118
        {
2119
          "datasource": {
2120
            "uid": "$datasource"
2121
          },
2122
          "expr": "(\n  sum by (pod) (kube_pod_container_status_terminated{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n) * on(pod) group_left(reason) (\n  sum by (pod, reason) (kube_pod_container_status_terminated_reason{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n)",
2123
          "interval": "",
2124
          "legendFormat": "{{pod}}  - TERMINATED -> {{reason}}",
2125
          "queryType": "randomWalk",
2126
          "refId": "B"
2127
        },
2128
        {
2129
          "datasource": {
2130
            "uid": "$datasource"
2131
          },
2132
          "expr": "(\n  sum by (pod) (kube_pod_container_status_waiting{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n) * on(pod) group_left(reason) (\n  sum by (pod, reason) (kube_pod_container_status_waiting_reason{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n)",
2133
          "interval": "",
2134
          "legendFormat": "{{pod}}  - WAITING -> {{reason}}",
2135
          "queryType": "randomWalk",
2136
          "refId": "C"
2137
        }
2138
      ],
2139
      "thresholds": [],
2140
      "timeRegions": [],
2141
      "title": "Pod Status",
2142
      "tooltip": {
2143
        "shared": true,
2144
        "sort": 2,
2145
        "value_type": "individual"
2146
      },
2147
      "type": "graph",
2148
      "xaxis": {
2149
        "mode": "time",
2150
        "show": true,
2151
        "values": []
2152
      },
2153
      "yaxes": [
2154
        {
2155
          "decimals": 0,
2156
          "format": "short",
2157
          "logBase": 1,
2158
          "show": true
2159
        },
2160
        {
2161
          "format": "short",
2162
          "logBase": 1,
2163
          "show": true
2164
        }
2165
      ],
2166
      "yaxis": {
2167
        "align": false
2168
      }
2169
    },
2170
    {
2171
      "aliasColors": {},
2172
      "bars": false,
2173
      "dashLength": 10,
2174
      "dashes": false,
2175
      "datasource": {
2176
        "uid": "$datasource"
2177
      },
2178
      "decimals": 0,
2179
      "fill": 1,
2180
      "fillGradient": 0,
2181
      "gridPos": {
2182
        "h": 9,
2183
        "w": 8,
2184
        "x": 16,
2185
        "y": 51
2186
      },
2187
      "hiddenSeries": false,
2188
      "id": 40,
2189
      "legend": {
2190
        "alignAsTable": true,
2191
        "avg": false,
2192
        "current": true,
2193
        "max": false,
2194
        "min": false,
2195
        "show": true,
2196
        "total": false,
2197
        "values": true
2198
      },
2199
      "lines": true,
2200
      "linewidth": 1,
2201
      "nullPointMode": "null",
2202
      "options": {
2203
        "alertThreshold": true
2204
      },
2205
      "percentage": false,
2206
      "pluginVersion": "9.4.3",
2207
      "pointradius": 2,
2208
      "points": false,
2209
      "renderer": "flot",
2210
      "seriesOverrides": [],
2211
      "spaceLength": 10,
2212
      "stack": false,
2213
      "steppedLine": false,
2214
      "targets": [
2215
        {
2216
          "datasource": {
2217
            "uid": "$datasource"
2218
          },
2219
          "expr": "kube_daemonset_status_desired_number_scheduled{cluster=~\"$cluster\", daemonset=\"ws-daemon\"}",
2220
          "interval": "",
2221
          "legendFormat": "{{cluster}} - {{daemonset}} - Desired",
2222
          "queryType": "randomWalk",
2223
          "refId": "C"
2224
        },
2225
        {
2226
          "datasource": {
2227
            "uid": "$datasource"
2228
          },
2229
          "expr": "kube_daemonset_status_number_available{cluster=~\"$cluster\", daemonset=\"ws-daemon\"}",
2230
          "interval": "",
2231
          "legendFormat": "{{cluster}} - {{daemonset}} - Available replicas",
2232
          "queryType": "randomWalk",
2233
          "refId": "A"
2234
        },
2235
        {
2236
          "datasource": {
2237
            "uid": "$datasource"
2238
          },
2239
          "expr": "kube_daemonset_status_number_unavailable{cluster=~\"$cluster\", daemonset=\"ws-daemon\"}",
2240
          "interval": "",
2241
          "legendFormat": "{{cluster}} - {{daemonset}} - Unvailable replicas",
2242
          "queryType": "randomWalk",
2243
          "refId": "B"
2244
        }
2245
      ],
2246
      "thresholds": [],
2247
      "timeRegions": [],
2248
      "title": "Replicas availability",
2249
      "tooltip": {
2250
        "shared": true,
2251
        "sort": 2,
2252
        "value_type": "individual"
2253
      },
2254
      "type": "graph",
2255
      "xaxis": {
2256
        "mode": "time",
2257
        "show": true,
2258
        "values": []
2259
      },
2260
      "yaxes": [
2261
        {
2262
          "decimals": 0,
2263
          "format": "short",
2264
          "logBase": 1,
2265
          "show": true
2266
        },
2267
        {
2268
          "format": "short",
2269
          "logBase": 1,
2270
          "show": true
2271
        }
2272
      ],
2273
      "yaxis": {
2274
        "align": false
2275
      }
2276
    },
2277
    {
2278
      "collapsed": true,
2279
      "datasource": {
2280
        "type": "datasource",
2281
        "uid": "grafana"
2282
      },
2283
      "gridPos": {
2284
        "h": 1,
2285
        "w": 24,
2286
        "x": 0,
2287
        "y": 60
2288
      },
2289
      "id": 22,
2290
      "panels": [
2291
        {
2292
          "aliasColors": {},
2293
          "bars": false,
2294
          "dashLength": 10,
2295
          "dashes": false,
2296
          "datasource": {
2297
            "uid": "$datasource"
2298
          },
2299
          "fieldConfig": {
2300
            "defaults": {
2301
              "custom": {}
2302
            },
2303
            "overrides": []
2304
          },
2305
          "fill": 1,
2306
          "fillGradient": 0,
2307
          "gridPos": {
2308
            "h": 9,
2309
            "w": 12,
2310
            "x": 0,
2311
            "y": 59
2312
          },
2313
          "hiddenSeries": false,
2314
          "id": 32,
2315
          "legend": {
2316
            "alignAsTable": true,
2317
            "avg": true,
2318
            "current": true,
2319
            "max": true,
2320
            "min": false,
2321
            "show": true,
2322
            "total": false,
2323
            "values": true
2324
          },
2325
          "lines": true,
2326
          "linewidth": 1,
2327
          "nullPointMode": "null",
2328
          "options": {
2329
            "alertThreshold": true
2330
          },
2331
          "percentage": false,
2332
          "pluginVersion": "7.4.3",
2333
          "pointradius": 2,
2334
          "points": false,
2335
          "renderer": "flot",
2336
          "seriesOverrides": [],
2337
          "spaceLength": 10,
2338
          "stack": false,
2339
          "steppedLine": false,
2340
          "targets": [
2341
            {
2342
              "datasource": {
2343
                "uid": "$datasource"
2344
              },
2345
              "expr": "process_resident_memory_bytes{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\"}",
2346
              "interval": "",
2347
              "legendFormat": "{{cluster}} - {{pod}}",
2348
              "queryType": "randomWalk",
2349
              "refId": "A"
2350
            }
2351
          ],
2352
          "thresholds": [],
2353
          "timeRegions": [],
2354
          "title": "Memory Usage (as seen by the runtime process)",
2355
          "tooltip": {
2356
            "shared": true,
2357
            "sort": 2,
2358
            "value_type": "individual"
2359
          },
2360
          "type": "graph",
2361
          "xaxis": {
2362
            "mode": "time",
2363
            "show": true,
2364
            "values": []
2365
          },
2366
          "yaxes": [
2367
            {
2368
              "format": "bytes",
2369
              "logBase": 1,
2370
              "min": "0",
2371
              "show": true
2372
            },
2373
            {
2374
              "format": "short",
2375
              "logBase": 1,
2376
              "show": true
2377
            }
2378
          ],
2379
          "yaxis": {
2380
            "align": false
2381
          }
2382
        },
2383
        {
2384
          "aliasColors": {},
2385
          "bars": false,
2386
          "dashLength": 10,
2387
          "dashes": false,
2388
          "datasource": {
2389
            "uid": "$datasource"
2390
          },
2391
          "decimals": 2,
2392
          "editable": true,
2393
          "error": false,
2394
          "fieldConfig": {
2395
            "defaults": {
2396
              "custom": {}
2397
            },
2398
            "overrides": []
2399
          },
2400
          "fill": 1,
2401
          "fillGradient": 0,
2402
          "gridPos": {
2403
            "h": 9,
2404
            "w": 12,
2405
            "x": 12,
2406
            "y": 59
2407
          },
2408
          "hiddenSeries": false,
2409
          "id": 34,
2410
          "legend": {
2411
            "alignAsTable": true,
2412
            "avg": true,
2413
            "current": true,
2414
            "hideEmpty": false,
2415
            "max": true,
2416
            "min": false,
2417
            "rightSide": false,
2418
            "show": true,
2419
            "total": false,
2420
            "values": true
2421
          },
2422
          "lines": true,
2423
          "linewidth": 1,
2424
          "links": [],
2425
          "nullPointMode": "null",
2426
          "options": {
2427
            "alertThreshold": true
2428
          },
2429
          "percentage": false,
2430
          "pluginVersion": "7.4.3",
2431
          "pointradius": 5,
2432
          "points": false,
2433
          "renderer": "flot",
2434
          "seriesOverrides": [],
2435
          "spaceLength": 10,
2436
          "stack": false,
2437
          "steppedLine": false,
2438
          "targets": [
2439
            {
2440
              "datasource": {
2441
                "uid": "$datasource"
2442
              },
2443
              "expr": "rate(process_cpu_seconds_total{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\"}[5m])",
2444
              "interval": "",
2445
              "intervalFactor": 2,
2446
              "legendFormat": "{{cluster}} - {{pod}}",
2447
              "metric": "prometheus_local_storage_ingested_samples_total",
2448
              "refId": "A",
2449
              "step": 10
2450
            }
2451
          ],
2452
          "thresholds": [],
2453
          "timeRegions": [],
2454
          "title": "CPU Usage (as seen by the runtime process)",
2455
          "tooltip": {
2456
            "msResolution": false,
2457
            "shared": true,
2458
            "sort": 2,
2459
            "value_type": "individual"
2460
          },
2461
          "type": "graph",
2462
          "xaxis": {
2463
            "mode": "time",
2464
            "show": true,
2465
            "values": [
2466
              "avg"
2467
            ]
2468
          },
2469
          "yaxes": [
2470
            {
2471
              "decimals": 2,
2472
              "format": "none",
2473
              "logBase": 1,
2474
              "min": "0",
2475
              "show": true
2476
            },
2477
            {
2478
              "format": "short",
2479
              "logBase": 1,
2480
              "show": true
2481
            }
2482
          ],
2483
          "yaxis": {
2484
            "align": false
2485
          }
2486
        },
2487
        {
2488
          "aliasColors": {},
2489
          "bars": false,
2490
          "dashLength": 10,
2491
          "dashes": false,
2492
          "datasource": {
2493
            "uid": "$datasource"
2494
          },
2495
          "fieldConfig": {
2496
            "defaults": {
2497
              "custom": {}
2498
            },
2499
            "overrides": []
2500
          },
2501
          "fill": 1,
2502
          "fillGradient": 0,
2503
          "gridPos": {
2504
            "h": 9,
2505
            "w": 12,
2506
            "x": 0,
2507
            "y": 68
2508
          },
2509
          "hiddenSeries": false,
2510
          "id": 26,
2511
          "legend": {
2512
            "alignAsTable": true,
2513
            "avg": true,
2514
            "current": true,
2515
            "max": true,
2516
            "min": false,
2517
            "show": true,
2518
            "total": false,
2519
            "values": true
2520
          },
2521
          "lines": true,
2522
          "linewidth": 1,
2523
          "nullPointMode": "null",
2524
          "options": {
2525
            "alertThreshold": true
2526
          },
2527
          "percentage": false,
2528
          "pluginVersion": "7.4.3",
2529
          "pointradius": 2,
2530
          "points": false,
2531
          "renderer": "flot",
2532
          "seriesOverrides": [],
2533
          "spaceLength": 10,
2534
          "stack": false,
2535
          "steppedLine": false,
2536
          "targets": [
2537
            {
2538
              "datasource": {
2539
                "uid": "$datasource"
2540
              },
2541
              "expr": "go_memstats_heap_sys_bytes{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\"}",
2542
              "interval": "",
2543
              "legendFormat": "{{cluster}} - {{pod}}",
2544
              "queryType": "randomWalk",
2545
              "refId": "A"
2546
            }
2547
          ],
2548
          "thresholds": [],
2549
          "timeRegions": [],
2550
          "title": "Heap Usage",
2551
          "tooltip": {
2552
            "shared": true,
2553
            "sort": 2,
2554
            "value_type": "individual"
2555
          },
2556
          "type": "graph",
2557
          "xaxis": {
2558
            "mode": "time",
2559
            "show": true,
2560
            "values": []
2561
          },
2562
          "yaxes": [
2563
            {
2564
              "format": "bytes",
2565
              "logBase": 1,
2566
              "min": "0",
2567
              "show": true
2568
            },
2569
            {
2570
              "format": "short",
2571
              "logBase": 1,
2572
              "show": true
2573
            }
2574
          ],
2575
          "yaxis": {
2576
            "align": false
2577
          }
2578
        },
2579
        {
2580
          "aliasColors": {},
2581
          "bars": false,
2582
          "dashLength": 10,
2583
          "dashes": false,
2584
          "datasource": {
2585
            "uid": "$datasource"
2586
          },
2587
          "fieldConfig": {
2588
            "defaults": {
2589
              "custom": {},
2590
              "links": []
2591
            },
2592
            "overrides": []
2593
          },
2594
          "fill": 1,
2595
          "fillGradient": 0,
2596
          "gridPos": {
2597
            "h": 9,
2598
            "w": 12,
2599
            "x": 12,
2600
            "y": 68
2601
          },
2602
          "hiddenSeries": false,
2603
          "id": 48,
2604
          "legend": {
2605
            "alignAsTable": true,
2606
            "avg": true,
2607
            "current": true,
2608
            "max": true,
2609
            "min": false,
2610
            "show": true,
2611
            "total": false,
2612
            "values": true
2613
          },
2614
          "lines": true,
2615
          "linewidth": 1,
2616
          "nullPointMode": "null",
2617
          "options": {
2618
            "alertThreshold": true
2619
          },
2620
          "percentage": false,
2621
          "pluginVersion": "7.4.3",
2622
          "pointradius": 2,
2623
          "points": false,
2624
          "renderer": "flot",
2625
          "repeatDirection": "h",
2626
          "seriesOverrides": [
2627
            {
2628
              "alias": "/GC rate:(.*)/",
2629
              "yaxis": 2
2630
            }
2631
          ],
2632
          "spaceLength": 10,
2633
          "stack": false,
2634
          "steppedLine": false,
2635
          "targets": [
2636
            {
2637
              "datasource": {
2638
                "uid": "$datasource"
2639
              },
2640
              "expr": "rate(go_memstats_alloc_bytes_total{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\"}[5m])",
2641
              "interval": "",
2642
              "legendFormat": "{{cluster}} - {{pod}}",
2643
              "refId": "A"
2644
            }
2645
          ],
2646
          "thresholds": [],
2647
          "timeRegions": [],
2648
          "title": "Memory Allocation rate",
2649
          "tooltip": {
2650
            "shared": true,
2651
            "sort": 2,
2652
            "value_type": "individual"
2653
          },
2654
          "type": "graph",
2655
          "xaxis": {
2656
            "mode": "time",
2657
            "show": true,
2658
            "values": []
2659
          },
2660
          "yaxes": [
2661
            {
2662
              "format": "binBps",
2663
              "logBase": 1,
2664
              "min": "0",
2665
              "show": true
2666
            },
2667
            {
2668
              "format": "short",
2669
              "logBase": 1,
2670
              "show": true
2671
            }
2672
          ],
2673
          "yaxis": {
2674
            "align": false
2675
          }
2676
        },
2677
        {
2678
          "aliasColors": {},
2679
          "bars": false,
2680
          "dashLength": 10,
2681
          "dashes": false,
2682
          "datasource": {
2683
            "uid": "$datasource"
2684
          },
2685
          "fieldConfig": {
2686
            "defaults": {
2687
              "custom": {}
2688
            },
2689
            "overrides": []
2690
          },
2691
          "fill": 1,
2692
          "fillGradient": 0,
2693
          "gridPos": {
2694
            "h": 9,
2695
            "w": 12,
2696
            "x": 0,
2697
            "y": 77
2698
          },
2699
          "hiddenSeries": false,
2700
          "id": 24,
2701
          "legend": {
2702
            "alignAsTable": true,
2703
            "avg": true,
2704
            "current": true,
2705
            "max": true,
2706
            "min": false,
2707
            "show": true,
2708
            "total": false,
2709
            "values": true
2710
          },
2711
          "lines": true,
2712
          "linewidth": 1,
2713
          "nullPointMode": "null",
2714
          "options": {
2715
            "alertThreshold": true
2716
          },
2717
          "percentage": false,
2718
          "pluginVersion": "7.4.3",
2719
          "pointradius": 2,
2720
          "points": false,
2721
          "renderer": "flot",
2722
          "seriesOverrides": [],
2723
          "spaceLength": 10,
2724
          "stack": false,
2725
          "steppedLine": false,
2726
          "targets": [
2727
            {
2728
              "datasource": {
2729
                "uid": "$datasource"
2730
              },
2731
              "expr": "go_goroutines{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\"}",
2732
              "interval": "",
2733
              "legendFormat": "{{cluster}} - {{pod}}",
2734
              "queryType": "randomWalk",
2735
              "refId": "A"
2736
            }
2737
          ],
2738
          "thresholds": [],
2739
          "timeRegions": [],
2740
          "title": "Go Routines",
2741
          "tooltip": {
2742
            "shared": true,
2743
            "sort": 2,
2744
            "value_type": "individual"
2745
          },
2746
          "type": "graph",
2747
          "xaxis": {
2748
            "mode": "time",
2749
            "show": true,
2750
            "values": []
2751
          },
2752
          "yaxes": [
2753
            {
2754
              "format": "short",
2755
              "logBase": 1,
2756
              "min": "0",
2757
              "show": true
2758
            },
2759
            {
2760
              "format": "short",
2761
              "logBase": 1,
2762
              "show": true
2763
            }
2764
          ],
2765
          "yaxis": {
2766
            "align": false
2767
          }
2768
        },
2769
        {
2770
          "aliasColors": {},
2771
          "bars": false,
2772
          "dashLength": 10,
2773
          "dashes": false,
2774
          "datasource": {
2775
            "uid": "$datasource"
2776
          },
2777
          "fieldConfig": {
2778
            "defaults": {
2779
              "custom": {}
2780
            },
2781
            "overrides": []
2782
          },
2783
          "fill": 1,
2784
          "fillGradient": 0,
2785
          "gridPos": {
2786
            "h": 9,
2787
            "w": 12,
2788
            "x": 12,
2789
            "y": 77
2790
          },
2791
          "hiddenSeries": false,
2792
          "id": 50,
2793
          "legend": {
2794
            "alignAsTable": true,
2795
            "avg": true,
2796
            "current": true,
2797
            "max": true,
2798
            "min": false,
2799
            "show": true,
2800
            "total": false,
2801
            "values": true
2802
          },
2803
          "lines": true,
2804
          "linewidth": 1,
2805
          "nullPointMode": "null",
2806
          "options": {
2807
            "alertThreshold": true
2808
          },
2809
          "percentage": false,
2810
          "pluginVersion": "7.4.3",
2811
          "pointradius": 2,
2812
          "points": false,
2813
          "renderer": "flot",
2814
          "seriesOverrides": [],
2815
          "spaceLength": 10,
2816
          "stack": false,
2817
          "steppedLine": false,
2818
          "targets": [
2819
            {
2820
              "datasource": {
2821
                "uid": "$datasource"
2822
              },
2823
              "expr": "go_gc_duration_seconds{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\", quantile=\"0.5\"}",
2824
              "interval": "",
2825
              "legendFormat": "{{cluster}} - {{pod}} - GC Duration - 50th percentile",
2826
              "refId": "A"
2827
            },
2828
            {
2829
              "datasource": {
2830
                "uid": "$datasource"
2831
              },
2832
              "expr": "go_gc_duration_seconds{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\", quantile=\"0.75\"}",
2833
              "interval": "",
2834
              "legendFormat": "{{cluster}} - {{pod}} - GC Duration - 75th percentile",
2835
              "refId": "B"
2836
            },
2837
            {
2838
              "datasource": {
2839
                "uid": "$datasource"
2840
              },
2841
              "expr": "go_gc_duration_seconds{cluster=~\"$cluster\", job=\"ws-daemon\", pod=~\"$pod\", quantile=\"1\"}",
2842
              "interval": "",
2843
              "legendFormat": "{{cluster}} - {{pod}} - GC Duration - 100th percentile",
2844
              "refId": "C"
2845
            }
2846
          ],
2847
          "thresholds": [],
2848
          "timeRegions": [],
2849
          "title": "Garbage collection time",
2850
          "tooltip": {
2851
            "shared": true,
2852
            "sort": 2,
2853
            "value_type": "individual"
2854
          },
2855
          "type": "graph",
2856
          "xaxis": {
2857
            "mode": "time",
2858
            "show": true,
2859
            "values": []
2860
          },
2861
          "yaxes": [
2862
            {
2863
              "format": "s",
2864
              "logBase": 1,
2865
              "min": "0",
2866
              "show": true
2867
            },
2868
            {
2869
              "format": "short",
2870
              "logBase": 1,
2871
              "show": true
2872
            }
2873
          ],
2874
          "yaxis": {
2875
            "align": false
2876
          }
2877
        }
2878
      ],
2879
      "targets": [
2880
        {
2881
          "datasource": {
2882
            "type": "datasource",
2883
            "uid": "grafana"
2884
          },
2885
          "refId": "A"
2886
        }
2887
      ],
2888
      "title": "Go Runtime Metrics",
2889
      "type": "row"
2890
    }
2891
  ],
2892
  "refresh": "30s",
2893
  "revision": 1,
2894
  "schemaVersion": 38,
2895
  "style": "dark",
2896
  "tags": [
2897
    "gitpod-mixin"
2898
  ],
2899
  "templating": {
2900
    "list": [
2901
      {
2902
        "current": {
2903
          "selected": false,
2904
          "text": "All",
2905
          "value": "$__all"
2906
        },
2907
        "datasource": {
2908
          "type": "prometheus",
2909
          "uid": "$datasource"
2910
        },
2911
        "definition": "label_values(up{job=\"ws-daemon\"}, cluster)",
2912
        "hide": 0,
2913
        "includeAll": true,
2914
        "label": "Cluster",
2915
        "multi": true,
2916
        "name": "cluster",
2917
        "options": [],
2918
        "query": {
2919
          "query": "label_values(up{job=\"ws-daemon\"}, cluster)",
2920
          "refId": "StandardVariableQuery"
2921
        },
2922
        "refresh": 2,
2923
        "regex": "",
2924
        "skipUrlSync": false,
2925
        "sort": 0,
2926
        "tagValuesQuery": "",
2927
        "tagsQuery": "",
2928
        "type": "query",
2929
        "useTags": false
2930
      },
2931
      {
2932
        "current": {
2933
          "selected": true,
2934
          "text": [
2935
            "All"
2936
          ],
2937
          "value": [
2938
            "$__all"
2939
          ]
2940
        },
2941
        "datasource": {
2942
          "type": "prometheus",
2943
          "uid": "$datasource"
2944
        },
2945
        "definition": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", pod=~\"ws-daemon.*\"}, node)",
2946
        "hide": 0,
2947
        "includeAll": true,
2948
        "label": "Node",
2949
        "multi": true,
2950
        "name": "node",
2951
        "options": [],
2952
        "query": {
2953
          "query": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", pod=~\"ws-daemon.*\"}, node)",
2954
          "refId": "StandardVariableQuery"
2955
        },
2956
        "refresh": 2,
2957
        "regex": "",
2958
        "skipUrlSync": false,
2959
        "sort": 0,
2960
        "tagValuesQuery": "",
2961
        "tagsQuery": "",
2962
        "type": "query",
2963
        "useTags": false
2964
      },
2965
      {
2966
        "current": {
2967
          "selected": true,
2968
          "text": [
2969
            "All"
2970
          ],
2971
          "value": [
2972
            "$__all"
2973
          ]
2974
        },
2975
        "datasource": {
2976
          "type": "prometheus",
2977
          "uid": "$datasource"
2978
        },
2979
        "definition": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", node=~\"$node\", pod=~\"ws-daemon.*\"}, pod)",
2980
        "hide": 0,
2981
        "includeAll": true,
2982
        "label": "Pod",
2983
        "multi": true,
2984
        "name": "pod",
2985
        "options": [],
2986
        "query": {
2987
          "query": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", node=~\"$node\", pod=~\"ws-daemon.*\"}, pod)",
2988
          "refId": "StandardVariableQuery"
2989
        },
2990
        "refresh": 2,
2991
        "regex": "",
2992
        "skipUrlSync": false,
2993
        "sort": 0,
2994
        "tagValuesQuery": "",
2995
        "tagsQuery": "",
2996
        "type": "query",
2997
        "useTags": false
2998
      },
2999
      {
3000
        "current": {
3001
          "selected": true,
3002
          "text": [
3003
            "All"
3004
          ],
3005
          "value": [
3006
            "$__all"
3007
          ]
3008
        },
3009
        "datasource": {
3010
          "type": "prometheus",
3011
          "uid": "$datasource"
3012
        },
3013
        "definition": "label_values(grpc_server_handled_total{job=\"ws-daemon\", cluster=~\"$cluster\"}, grpc_method)",
3014
        "hide": 0,
3015
        "includeAll": true,
3016
        "label": "gRPC Method",
3017
        "multi": true,
3018
        "name": "grpc_method",
3019
        "options": [],
3020
        "query": {
3021
          "query": "label_values(grpc_server_handled_total{job=\"ws-daemon\", cluster=~\"$cluster\"}, grpc_method)",
3022
          "refId": "StandardVariableQuery"
3023
        },
3024
        "refresh": 2,
3025
        "regex": "",
3026
        "skipUrlSync": false,
3027
        "sort": 0,
3028
        "tagValuesQuery": "",
3029
        "tagsQuery": "",
3030
        "type": "query",
3031
        "useTags": false
3032
      },
3033
      {
3034
        "current": {
3035
          "selected": false,
3036
          "text": "VictoriaMetrics",
3037
          "value": "VictoriaMetrics"
3038
        },
3039
        "hide": 2,
3040
        "includeAll": false,
3041
        "multi": false,
3042
        "name": "datasource",
3043
        "options": [],
3044
        "query": "prometheus",
3045
        "refresh": 2,
3046
        "regex": "",
3047
        "skipUrlSync": false,
3048
        "type": "datasource"
3049
      },
3050
      {
3051
        "current": {
3052
          "selected": true,
3053
          "text": [
3054
            "All"
3055
          ],
3056
          "value": [
3057
            "$__all"
3058
          ]
3059
        },
3060
        "datasource": {
3061
          "type": "prometheus",
3062
          "uid": "${datasource}"
3063
        },
3064
        "definition": "label_values(workqueue_depth{pod=~\"$pod\"}, name)",
3065
        "hide": 0,
3066
        "includeAll": true,
3067
        "label": "Controller",
3068
        "multi": true,
3069
        "name": "controller",
3070
        "options": [],
3071
        "query": {
3072
          "query": "label_values(workqueue_depth{pod=~\"$pod\"}, name)",
3073
          "refId": "StandardVariableQuery"
3074
        },
3075
        "refresh": 1,
3076
        "regex": "",
3077
        "skipUrlSync": false,
3078
        "sort": 0,
3079
        "type": "query"
3080
      }
3081
    ]
3082
  },
3083
  "time": {
3084
    "from": "now-1h",
3085
    "to": "now"
3086
  },
3087
  "timepicker": {},
3088
  "timezone": "utc",
3089
  "title": "Gitpod / Component / ws-daemon",
3090
  "uid": "ws-daemon",
3091
  "version": 1,
3092
  "weekStart": ""
3093
}
3094

3095
Product

Resources

Company
