CoCalc -- ws-manager-mk2.json

GitHub Repository: gitpod-io/gitpod
Path: blob/main/operations/observability/mixins/workspace/dashboards/components/ws-manager-mk2.json
²⁵⁰⁶ views
1
{
2
  "annotations": {
3
    "list": [
4
      {
5
        "builtIn": 1,
6
        "datasource": {
7
          "type": "datasource",
8
          "uid": "grafana"
9
        },
10
        "enable": true,
11
        "hide": true,
12
        "iconColor": "rgba(0, 211, 255, 1)",
13
        "name": "Annotations & Alerts",
14
        "target": {
15
          "limit": 100,
16
          "matchAny": false,
17
          "tags": [],
18
          "type": "dashboard"
19
        },
20
        "type": "dashboard"
21
      }
22
    ]
23
  },
24
  "editable": true,
25
  "fiscalYearStartMonth": 0,
26
  "graphTooltip": 1,
27
  "id": 76,
28
  "links": [],
29
  "liveNow": false,
30
  "panels": [
31
    {
32
      "collapsed": false,
33
      "datasource": {
34
        "type": "datasource",
35
        "uid": "grafana"
36
      },
37
      "gridPos": {
38
        "h": 1,
39
        "w": 24,
40
        "x": 0,
41
        "y": 0
42
      },
43
      "id": 42,
44
      "panels": [],
45
      "targets": [
46
        {
47
          "datasource": {
48
            "type": "datasource",
49
            "uid": "grafana"
50
          },
51
          "refId": "A"
52
        }
53
      ],
54
      "title": "ws-manager-mk2 Metrics",
55
      "type": "row"
56
    },
57
    {
58
      "datasource": {
59
        "type": "prometheus",
60
        "uid": "$datasource"
61
      },
62
      "fieldConfig": {
63
        "defaults": {
64
          "color": {
65
            "mode": "palette-classic"
66
          },
67
          "custom": {
68
            "axisCenteredZero": false,
69
            "axisColorMode": "text",
70
            "axisLabel": "",
71
            "axisPlacement": "auto",
72
            "barAlignment": 0,
73
            "drawStyle": "line",
74
            "fillOpacity": 10,
75
            "gradientMode": "none",
76
            "hideFrom": {
77
              "legend": false,
78
              "tooltip": false,
79
              "viz": false
80
            },
81
            "lineInterpolation": "linear",
82
            "lineWidth": 1,
83
            "pointSize": 5,
84
            "scaleDistribution": {
85
              "type": "linear"
86
            },
87
            "showPoints": "never",
88
            "spanNulls": false,
89
            "stacking": {
90
              "group": "A",
91
              "mode": "none"
92
            },
93
            "thresholdsStyle": {
94
              "mode": "off"
95
            }
96
          },
97
          "decimals": 0,
98
          "links": [],
99
          "mappings": [],
100
          "min": 0,
101
          "thresholds": {
102
            "mode": "absolute",
103
            "steps": [
104
              {
105
                "color": "green",
106
                "value": null
107
              },
108
              {
109
                "color": "red",
110
                "value": 80
111
              }
112
            ]
113
          },
114
          "unit": "short"
115
        },
116
        "overrides": [
117
          {
118
            "matcher": {
119
              "id": "byRegexp",
120
              "options": ".*Running"
121
            },
122
            "properties": [
123
              {
124
                "id": "color",
125
                "value": {
126
                  "fixedColor": "green",
127
                  "mode": "fixed"
128
                }
129
              }
130
            ]
131
          },
132
          {
133
            "matcher": {
134
              "id": "byRegexp",
135
              "options": ".*Pending"
136
            },
137
            "properties": [
138
              {
139
                "id": "color",
140
                "value": {
141
                  "fixedColor": "super-light-blue",
142
                  "mode": "fixed"
143
                }
144
              }
145
            ]
146
          },
147
          {
148
            "matcher": {
149
              "id": "byRegexp",
150
              "options": ".*Creating"
151
            },
152
            "properties": [
153
              {
154
                "id": "color",
155
                "value": {
156
                  "fixedColor": "blue",
157
                  "mode": "fixed"
158
                }
159
              }
160
            ]
161
          },
162
          {
163
            "matcher": {
164
              "id": "byRegexp",
165
              "options": ".*Initializing"
166
            },
167
            "properties": [
168
              {
169
                "id": "color",
170
                "value": {
171
                  "fixedColor": "yellow",
172
                  "mode": "fixed"
173
                }
174
              }
175
            ]
176
          },
177
          {
178
            "matcher": {
179
              "id": "byRegexp",
180
              "options": ".*Stopping"
181
            },
182
            "properties": [
183
              {
184
                "id": "color",
185
                "value": {
186
                  "fixedColor": "red",
187
                  "mode": "fixed"
188
                }
189
              }
190
            ]
191
          },
192
          {
193
            "matcher": {
194
              "id": "byRegexp",
195
              "options": ".*Stopped"
196
            },
197
            "properties": [
198
              {
199
                "id": "color",
200
                "value": {
201
                  "fixedColor": "#565656",
202
                  "mode": "fixed"
203
                }
204
              }
205
            ]
206
          }
207
        ]
208
      },
209
      "gridPos": {
210
        "h": 11,
211
        "w": 8,
212
        "x": 0,
213
        "y": 1
214
      },
215
      "id": 44,
216
      "options": {
217
        "legend": {
218
          "calcs": [
219
            "lastNotNull"
220
          ],
221
          "displayMode": "table",
222
          "placement": "bottom",
223
          "showLegend": true
224
        },
225
        "tooltip": {
226
          "mode": "multi",
227
          "sort": "desc"
228
        }
229
      },
230
      "pluginVersion": "9.4.3",
231
      "repeatDirection": "h",
232
      "targets": [
233
        {
234
          "datasource": {
235
            "uid": "$datasource"
236
          },
237
          "editorMode": "code",
238
          "expr": "sum(gitpod_ws_manager_mk2_workspace_phase_total{cluster=~\"$cluster\"}) by (type, cluster, phase)",
239
          "interval": "",
240
          "legendFormat": "{{cluster}} - {{ type }} - {{phase}}",
241
          "queryType": "randomWalk",
242
          "range": true,
243
          "refId": "A"
244
        }
245
      ],
246
      "title": "Workspace count",
247
      "type": "timeseries"
248
    },
249
    {
250
      "datasource": {
251
        "type": "prometheus",
252
        "uid": "$datasource"
253
      },
254
      "fieldConfig": {
255
        "defaults": {
256
          "color": {
257
            "mode": "palette-classic"
258
          },
259
          "custom": {
260
            "axisCenteredZero": false,
261
            "axisColorMode": "text",
262
            "axisLabel": "",
263
            "axisPlacement": "auto",
264
            "barAlignment": 0,
265
            "drawStyle": "line",
266
            "fillOpacity": 10,
267
            "gradientMode": "none",
268
            "hideFrom": {
269
              "legend": false,
270
              "tooltip": false,
271
              "viz": false
272
            },
273
            "lineInterpolation": "stepAfter",
274
            "lineWidth": 1,
275
            "pointSize": 5,
276
            "scaleDistribution": {
277
              "type": "linear"
278
            },
279
            "showPoints": "never",
280
            "spanNulls": false,
281
            "stacking": {
282
              "group": "A",
283
              "mode": "none"
284
            },
285
            "thresholdsStyle": {
286
              "mode": "off"
287
            }
288
          },
289
          "decimals": 2,
290
          "links": [],
291
          "mappings": [],
292
          "min": 0,
293
          "thresholds": {
294
            "mode": "absolute",
295
            "steps": [
296
              {
297
                "color": "green",
298
                "value": null
299
              },
300
              {
301
                "color": "red",
302
                "value": 80
303
              }
304
            ]
305
          },
306
          "unit": "ops"
307
        },
308
        "overrides": [
309
          {
310
            "matcher": {
311
              "id": "byRegexp",
312
              "options": "Failures: .*"
313
            },
314
            "properties": [
315
              {
316
                "id": "color",
317
                "value": {
318
                  "fixedColor": "red",
319
                  "mode": "fixed"
320
                }
321
              }
322
            ]
323
          }
324
        ]
325
      },
326
      "gridPos": {
327
        "h": 11,
328
        "w": 8,
329
        "x": 8,
330
        "y": 1
331
      },
332
      "id": 48,
333
      "links": [],
334
      "options": {
335
        "legend": {
336
          "calcs": [
337
            "mean",
338
            "lastNotNull",
339
            "max"
340
          ],
341
          "displayMode": "table",
342
          "placement": "bottom",
343
          "showLegend": true
344
        },
345
        "tooltip": {
346
          "mode": "multi",
347
          "sort": "desc"
348
        }
349
      },
350
      "pluginVersion": "9.3.6",
351
      "repeatDirection": "h",
352
      "targets": [
353
        {
354
          "datasource": {
355
            "uid": "$datasource"
356
          },
357
          "editorMode": "code",
358
          "expr": "sum(\n  rate(gitpod_ws_manager_mk2_workspace_startup_seconds_count{cluster=~\"$cluster\"}[1m])\n) by (cluster, type)",
359
          "hide": false,
360
          "interval": "",
361
          "legendFormat": "{{cluster}} - {{type}}",
362
          "range": true,
363
          "refId": "B"
364
        },
365
        {
366
          "datasource": {
367
            "type": "prometheus",
368
            "uid": "$datasource"
369
          },
370
          "editorMode": "code",
371
          "expr": "sum(rate(gitpod_ws_manager_mk2_workspace_starts_failure_total{cluster=~\"$cluster\"}[1m])) by (cluster, type)",
372
          "hide": false,
373
          "legendFormat": "Failures: {{cluster}} - {{type}}",
374
          "range": true,
375
          "refId": "A"
376
        }
377
      ],
378
      "title": "Workspace Starts",
379
      "type": "timeseries"
380
    },
381
    {
382
      "datasource": {
383
        "type": "prometheus",
384
        "uid": "$datasource"
385
      },
386
      "fieldConfig": {
387
        "defaults": {
388
          "color": {
389
            "mode": "palette-classic"
390
          },
391
          "custom": {
392
            "axisCenteredZero": false,
393
            "axisColorMode": "text",
394
            "axisLabel": "",
395
            "axisPlacement": "auto",
396
            "barAlignment": 0,
397
            "drawStyle": "line",
398
            "fillOpacity": 10,
399
            "gradientMode": "none",
400
            "hideFrom": {
401
              "legend": false,
402
              "tooltip": false,
403
              "viz": false
404
            },
405
            "lineInterpolation": "stepAfter",
406
            "lineWidth": 1,
407
            "pointSize": 5,
408
            "scaleDistribution": {
409
              "type": "linear"
410
            },
411
            "showPoints": "never",
412
            "spanNulls": false,
413
            "stacking": {
414
              "group": "A",
415
              "mode": "none"
416
            },
417
            "thresholdsStyle": {
418
              "mode": "off"
419
            }
420
          },
421
          "decimals": 2,
422
          "links": [],
423
          "mappings": [],
424
          "min": 0,
425
          "thresholds": {
426
            "mode": "absolute",
427
            "steps": [
428
              {
429
                "color": "green",
430
                "value": null
431
              },
432
              {
433
                "color": "red",
434
                "value": 80
435
              }
436
            ]
437
          },
438
          "unit": "ops"
439
        },
440
        "overrides": [
441
          {
442
            "matcher": {
443
              "id": "byRegexp",
444
              "options": ".* failed"
445
            },
446
            "properties": [
447
              {
448
                "id": "color",
449
                "value": {
450
                  "fixedColor": "red",
451
                  "mode": "fixed"
452
                }
453
              }
454
            ]
455
          },
456
          {
457
            "matcher": {
458
              "id": "byRegexp",
459
              "options": ".* regular-stop"
460
            },
461
            "properties": [
462
              {
463
                "id": "color",
464
                "value": {
465
                  "fixedColor": "green",
466
                  "mode": "fixed"
467
                }
468
              }
469
            ]
470
          }
471
        ]
472
      },
473
      "gridPos": {
474
        "h": 11,
475
        "w": 8,
476
        "x": 16,
477
        "y": 1
478
      },
479
      "id": 51,
480
      "links": [],
481
      "options": {
482
        "legend": {
483
          "calcs": [
484
            "mean",
485
            "lastNotNull",
486
            "max"
487
          ],
488
          "displayMode": "table",
489
          "placement": "bottom",
490
          "showLegend": true
491
        },
492
        "tooltip": {
493
          "mode": "multi",
494
          "sort": "desc"
495
        }
496
      },
497
      "pluginVersion": "9.4.3",
498
      "repeatDirection": "h",
499
      "targets": [
500
        {
501
          "datasource": {
502
            "uid": "$datasource"
503
          },
504
          "editorMode": "code",
505
          "expr": "sum(\n  rate(gitpod_ws_manager_mk2_workspace_stops_total{cluster=~\"$cluster\"}[$__rate_interval])\n) by (cluster, reason)",
506
          "interval": "",
507
          "legendFormat": "{{cluster}} - {{reason}}",
508
          "range": true,
509
          "refId": "C"
510
        }
511
      ],
512
      "title": "Workspace Stops",
513
      "type": "timeseries"
514
    },
515
    {
516
      "datasource": {
517
        "type": "prometheus",
518
        "uid": "$datasource"
519
      },
520
      "description": "Avg is shown here, just so we know that we are configuring our buckets right.\n\nMore info about how avg can help us with bucket configuration can be seen at Tom Wilkie's RED Method talk.",
521
      "fieldConfig": {
522
        "defaults": {
523
          "custom": {
524
            "hideFrom": {
525
              "legend": false,
526
              "tooltip": false,
527
              "viz": false
528
            },
529
            "scaleDistribution": {
530
              "type": "linear"
531
            }
532
          },
533
          "links": []
534
        },
535
        "overrides": []
536
      },
537
      "gridPos": {
538
        "h": 9,
539
        "w": 6,
540
        "x": 0,
541
        "y": 12
542
      },
543
      "id": 45,
544
      "links": [
545
        {
546
          "targetBlank": true,
547
          "title": "How avg help us configure representative Prometheus histogram buckets",
548
          "url": "https://youtu.be/zk77VS98Em8?t=908"
549
        }
550
      ],
551
      "options": {
552
        "calculate": false,
553
        "cellGap": 1,
554
        "color": {
555
          "exponent": 0.5,
556
          "fill": "dark-orange",
557
          "mode": "scheme",
558
          "reverse": false,
559
          "scale": "exponential",
560
          "scheme": "Greens",
561
          "steps": 64
562
        },
563
        "exemplars": {
564
          "color": "rgba(255,0,255,0.7)"
565
        },
566
        "filterValues": {
567
          "le": 1e-9
568
        },
569
        "legend": {
570
          "show": true
571
        },
572
        "rowsFrame": {
573
          "layout": "auto"
574
        },
575
        "tooltip": {
576
          "show": true,
577
          "yHistogram": false
578
        },
579
        "yAxis": {
580
          "axisPlacement": "left",
581
          "reverse": false,
582
          "unit": "s"
583
        }
584
      },
585
      "pluginVersion": "9.5.2",
586
      "repeatDirection": "h",
587
      "targets": [
588
        {
589
          "datasource": {
590
            "uid": "$datasource"
591
          },
592
          "editorMode": "code",
593
          "expr": "sum(\n    rate(gitpod_ws_manager_mk2_workspace_startup_seconds_bucket{cluster=~\"$cluster\", type=\"Regular\"}[$__rate_interval])\n  ) by (le)",
594
          "format": "heatmap",
595
          "interval": "",
596
          "legendFormat": "{{le}}",
597
          "queryType": "randomWalk",
598
          "range": true,
599
          "refId": "A"
600
        }
601
      ],
602
      "title": "Regular workspace startup time",
603
      "type": "heatmap"
604
    },
605
    {
606
      "datasource": {
607
        "type": "prometheus",
608
        "uid": "$datasource"
609
      },
610
      "description": "",
611
      "fieldConfig": {
612
        "defaults": {
613
          "custom": {
614
            "hideFrom": {
615
              "legend": false,
616
              "tooltip": false,
617
              "viz": false
618
            },
619
            "scaleDistribution": {
620
              "type": "linear"
621
            }
622
          },
623
          "links": []
624
        },
625
        "overrides": []
626
      },
627
      "gridPos": {
628
        "h": 9,
629
        "w": 6,
630
        "x": 6,
631
        "y": 12
632
      },
633
      "id": 166,
634
      "links": [
635
        {
636
          "targetBlank": true,
637
          "title": "How avg help us configure representative Prometheus histogram buckets",
638
          "url": "https://youtu.be/zk77VS98Em8?t=908"
639
        }
640
      ],
641
      "options": {
642
        "calculate": false,
643
        "cellGap": 1,
644
        "color": {
645
          "exponent": 0.5,
646
          "fill": "dark-orange",
647
          "mode": "scheme",
648
          "reverse": false,
649
          "scale": "exponential",
650
          "scheme": "Greens",
651
          "steps": 64
652
        },
653
        "exemplars": {
654
          "color": "rgba(255,0,255,0.7)"
655
        },
656
        "filterValues": {
657
          "le": 1e-9
658
        },
659
        "legend": {
660
          "show": true
661
        },
662
        "rowsFrame": {
663
          "layout": "auto"
664
        },
665
        "tooltip": {
666
          "show": true,
667
          "yHistogram": false
668
        },
669
        "yAxis": {
670
          "axisPlacement": "left",
671
          "reverse": false,
672
          "unit": "s"
673
        }
674
      },
675
      "pluginVersion": "9.5.2",
676
      "repeatDirection": "h",
677
      "targets": [
678
        {
679
          "datasource": {
680
            "uid": "$datasource"
681
          },
682
          "editorMode": "code",
683
          "expr": "sum(\n    rate(gitpod_ws_manager_mk2_workspace_startup_seconds_bucket{cluster=~\"$cluster\", type=\"ImageBuild\"}[$__rate_interval])\n  ) by (le)",
684
          "format": "heatmap",
685
          "interval": "",
686
          "legendFormat": "{{le}}",
687
          "queryType": "randomWalk",
688
          "range": true,
689
          "refId": "A"
690
        }
691
      ],
692
      "title": "ImageBuild startup time",
693
      "type": "heatmap"
694
    },
695
    {
696
      "datasource": {
697
        "type": "prometheus",
698
        "uid": "$datasource"
699
      },
700
      "description": "",
701
      "fieldConfig": {
702
        "defaults": {
703
          "custom": {
704
            "hideFrom": {
705
              "legend": false,
706
              "tooltip": false,
707
              "viz": false
708
            },
709
            "scaleDistribution": {
710
              "type": "linear"
711
            }
712
          },
713
          "links": []
714
        },
715
        "overrides": []
716
      },
717
      "gridPos": {
718
        "h": 9,
719
        "w": 6,
720
        "x": 12,
721
        "y": 12
722
      },
723
      "id": 172,
724
      "links": [
725
        {
726
          "targetBlank": true,
727
          "title": "How avg help us configure representative Prometheus histogram buckets",
728
          "url": "https://youtu.be/zk77VS98Em8?t=908"
729
        }
730
      ],
731
      "options": {
732
        "calculate": false,
733
        "cellGap": 1,
734
        "color": {
735
          "exponent": 0.5,
736
          "fill": "dark-orange",
737
          "mode": "scheme",
738
          "reverse": false,
739
          "scale": "exponential",
740
          "scheme": "Greens",
741
          "steps": 64
742
        },
743
        "exemplars": {
744
          "color": "rgba(255,0,255,0.7)"
745
        },
746
        "filterValues": {
747
          "le": 1e-9
748
        },
749
        "legend": {
750
          "show": true
751
        },
752
        "rowsFrame": {
753
          "layout": "auto"
754
        },
755
        "tooltip": {
756
          "show": true,
757
          "yHistogram": false
758
        },
759
        "yAxis": {
760
          "axisPlacement": "left",
761
          "reverse": false,
762
          "unit": "s"
763
        }
764
      },
765
      "pluginVersion": "9.5.2",
766
      "repeatDirection": "h",
767
      "targets": [
768
        {
769
          "datasource": {
770
            "uid": "$datasource"
771
          },
772
          "editorMode": "code",
773
          "expr": "sum(\n    rate(gitpod_ws_manager_mk2_workspace_startup_seconds_bucket{cluster=~\"$cluster\", type=\"Prebuild\"}[$__rate_interval])\n  ) by (le)",
774
          "format": "heatmap",
775
          "interval": "",
776
          "legendFormat": "{{le}}",
777
          "queryType": "randomWalk",
778
          "range": true,
779
          "refId": "A"
780
        }
781
      ],
782
      "title": "Prebuild startup time",
783
      "type": "heatmap"
784
    },
785
    {
786
      "datasource": {
787
        "type": "prometheus",
788
        "uid": "$datasource"
789
      },
790
      "fieldConfig": {
791
        "defaults": {
792
          "color": {
793
            "fixedColor": "red",
794
            "mode": "fixed"
795
          },
796
          "custom": {
797
            "axisCenteredZero": false,
798
            "axisColorMode": "text",
799
            "axisLabel": "",
800
            "axisPlacement": "auto",
801
            "barAlignment": 0,
802
            "drawStyle": "line",
803
            "fillOpacity": 0,
804
            "gradientMode": "none",
805
            "hideFrom": {
806
              "legend": false,
807
              "tooltip": false,
808
              "viz": false
809
            },
810
            "lineInterpolation": "linear",
811
            "lineWidth": 1,
812
            "pointSize": 5,
813
            "scaleDistribution": {
814
              "type": "linear"
815
            },
816
            "showPoints": "auto",
817
            "spanNulls": false,
818
            "stacking": {
819
              "group": "A",
820
              "mode": "none"
821
            },
822
            "thresholdsStyle": {
823
              "mode": "off"
824
            }
825
          },
826
          "links": [],
827
          "mappings": [],
828
          "min": 0,
829
          "thresholds": {
830
            "mode": "absolute",
831
            "steps": [
832
              {
833
                "color": "green",
834
                "value": null
835
              },
836
              {
837
                "color": "red",
838
                "value": 80
839
              }
840
            ]
841
          },
842
          "unit": "ops"
843
        },
844
        "overrides": []
845
      },
846
      "gridPos": {
847
        "h": 9,
848
        "w": 6,
849
        "x": 18,
850
        "y": 12
851
      },
852
      "id": 162,
853
      "links": [],
854
      "options": {
855
        "legend": {
856
          "calcs": [],
857
          "displayMode": "list",
858
          "placement": "bottom",
859
          "showLegend": true
860
        },
861
        "tooltip": {
862
          "mode": "multi",
863
          "sort": "none"
864
        }
865
      },
866
      "pluginVersion": "9.4.3",
867
      "repeatDirection": "h",
868
      "targets": [
869
        {
870
          "datasource": {
871
            "uid": "$datasource"
872
          },
873
          "editorMode": "code",
874
          "expr": "sum(\n  rate(gitpod_ws_manager_mk2_workspace_failure_total{cluster=~\"$cluster\"}[$__rate_interval])\n) by (cluster)",
875
          "interval": "",
876
          "legendFormat": "{{cluster}}",
877
          "range": true,
878
          "refId": "C"
879
        }
880
      ],
881
      "title": "Workspace Failures",
882
      "type": "timeseries"
883
    },
884
    {
885
      "collapsed": false,
886
      "gridPos": {
887
        "h": 1,
888
        "w": 24,
889
        "x": 0,
890
        "y": 21
891
      },
892
      "id": 88,
893
      "panels": [],
894
      "title": "Controller Manager",
895
      "type": "row"
896
    },
897
    {
898
      "aliasColors": {},
899
      "bars": false,
900
      "dashLength": 10,
901
      "dashes": false,
902
      "datasource": {
903
        "type": "prometheus",
904
        "uid": "$datasource"
905
      },
906
      "fieldConfig": {
907
        "defaults": {
908
          "links": []
909
        },
910
        "overrides": []
911
      },
912
      "fill": 1,
913
      "fillGradient": 0,
914
      "gridPos": {
915
        "h": 9,
916
        "w": 8,
917
        "x": 0,
918
        "y": 22
919
      },
920
      "hiddenSeries": false,
921
      "id": 100,
922
      "legend": {
923
        "alignAsTable": true,
924
        "avg": false,
925
        "current": true,
926
        "max": false,
927
        "min": false,
928
        "rightSide": false,
929
        "show": true,
930
        "total": false,
931
        "values": true
932
      },
933
      "lines": true,
934
      "linewidth": 1,
935
      "links": [],
936
      "nullPointMode": "null",
937
      "options": {
938
        "alertThreshold": true
939
      },
940
      "percentage": false,
941
      "pluginVersion": "9.5.2",
942
      "pointradius": 5,
943
      "points": false,
944
      "renderer": "flot",
945
      "seriesOverrides": [],
946
      "spaceLength": 10,
947
      "stack": false,
948
      "steppedLine": false,
949
      "targets": [
950
        {
951
          "datasource": {
952
            "uid": "$datasource"
953
          },
954
          "editorMode": "code",
955
          "expr": "sum(rate(workqueue_adds_total{job=\"ws-manager-mk2\", pod=~\"$pod\", name=~\"$controller\"}[5m])) by (cluster, pod, name)",
956
          "format": "time_series",
957
          "intervalFactor": 2,
958
          "legendFormat": "{{cluster}} {{pod}} {{name}}",
959
          "range": true,
960
          "refId": "A"
961
        }
962
      ],
963
      "thresholds": [],
964
      "timeRegions": [],
965
      "title": "Work Queue Add Rate",
966
      "tooltip": {
967
        "shared": false,
968
        "sort": 0,
969
        "value_type": "individual"
970
      },
971
      "type": "graph",
972
      "xaxis": {
973
        "mode": "time",
974
        "show": true,
975
        "values": []
976
      },
977
      "yaxes": [
978
        {
979
          "$$hashKey": "object:123",
980
          "format": "ops",
981
          "logBase": 1,
982
          "show": true
983
        },
984
        {
985
          "$$hashKey": "object:124",
986
          "format": "ops",
987
          "logBase": 1,
988
          "show": true
989
        }
990
      ],
991
      "yaxis": {
992
        "align": false
993
      }
994
    },
995
    {
996
      "aliasColors": {},
997
      "bars": false,
998
      "dashLength": 10,
999
      "dashes": false,
1000
      "datasource": {
1001
        "type": "prometheus",
1002
        "uid": "$datasource"
1003
      },
1004
      "fieldConfig": {
1005
        "defaults": {
1006
          "links": []
1007
        },
1008
        "overrides": []
1009
      },
1010
      "fill": 1,
1011
      "fillGradient": 0,
1012
      "gridPos": {
1013
        "h": 9,
1014
        "w": 8,
1015
        "x": 8,
1016
        "y": 22
1017
      },
1018
      "hiddenSeries": false,
1019
      "id": 147,
1020
      "legend": {
1021
        "alignAsTable": true,
1022
        "avg": false,
1023
        "current": true,
1024
        "max": false,
1025
        "min": false,
1026
        "rightSide": false,
1027
        "show": true,
1028
        "total": false,
1029
        "values": true
1030
      },
1031
      "lines": true,
1032
      "linewidth": 1,
1033
      "links": [],
1034
      "nullPointMode": "null",
1035
      "options": {
1036
        "alertThreshold": true
1037
      },
1038
      "percentage": false,
1039
      "pluginVersion": "9.5.2",
1040
      "pointradius": 5,
1041
      "points": false,
1042
      "renderer": "flot",
1043
      "seriesOverrides": [],
1044
      "spaceLength": 10,
1045
      "stack": false,
1046
      "steppedLine": false,
1047
      "targets": [
1048
        {
1049
          "datasource": {
1050
            "uid": "$datasource"
1051
          },
1052
          "editorMode": "code",
1053
          "expr": "sum(workqueue_depth{job=\"ws-manager-mk2\", pod=~\"$pod\", name=~\"$controller\"}) by (cluster, pod, name)",
1054
          "format": "time_series",
1055
          "intervalFactor": 2,
1056
          "legendFormat": "{{cluster}} {{pod}} {{name}}",
1057
          "range": true,
1058
          "refId": "A"
1059
        }
1060
      ],
1061
      "thresholds": [],
1062
      "timeRegions": [],
1063
      "title": "Work Queue Depth",
1064
      "tooltip": {
1065
        "shared": false,
1066
        "sort": 0,
1067
        "value_type": "individual"
1068
      },
1069
      "type": "graph",
1070
      "xaxis": {
1071
        "mode": "time",
1072
        "show": true,
1073
        "values": []
1074
      },
1075
      "yaxes": [
1076
        {
1077
          "$$hashKey": "object:123",
1078
          "format": "none",
1079
          "logBase": 1,
1080
          "min": "0",
1081
          "show": true
1082
        },
1083
        {
1084
          "$$hashKey": "object:124",
1085
          "format": "ops",
1086
          "logBase": 1,
1087
          "show": true
1088
        }
1089
      ],
1090
      "yaxis": {
1091
        "align": false
1092
      }
1093
    },
1094
    {
1095
      "datasource": {
1096
        "type": "prometheus",
1097
        "uid": "$datasource"
1098
      },
1099
      "fieldConfig": {
1100
        "defaults": {
1101
          "custom": {
1102
            "hideFrom": {
1103
              "legend": false,
1104
              "tooltip": false,
1105
              "viz": false
1106
            },
1107
            "scaleDistribution": {
1108
              "type": "linear"
1109
            }
1110
          },
1111
          "links": []
1112
        },
1113
        "overrides": []
1114
      },
1115
      "gridPos": {
1116
        "h": 9,
1117
        "w": 8,
1118
        "x": 16,
1119
        "y": 22
1120
      },
1121
      "id": 124,
1122
      "links": [],
1123
      "options": {
1124
        "calculate": false,
1125
        "cellGap": 1,
1126
        "color": {
1127
          "exponent": 0.5,
1128
          "fill": "dark-orange",
1129
          "mode": "scheme",
1130
          "reverse": false,
1131
          "scale": "exponential",
1132
          "scheme": "Spectral",
1133
          "steps": 64
1134
        },
1135
        "exemplars": {
1136
          "color": "rgba(255,0,255,0.7)"
1137
        },
1138
        "filterValues": {
1139
          "le": 1e-9
1140
        },
1141
        "legend": {
1142
          "show": true
1143
        },
1144
        "rowsFrame": {
1145
          "layout": "auto"
1146
        },
1147
        "tooltip": {
1148
          "show": true,
1149
          "yHistogram": false
1150
        },
1151
        "yAxis": {
1152
          "axisPlacement": "left",
1153
          "reverse": false,
1154
          "unit": "s"
1155
        }
1156
      },
1157
      "pluginVersion": "9.5.2",
1158
      "targets": [
1159
        {
1160
          "datasource": {
1161
            "uid": "$datasource"
1162
          },
1163
          "editorMode": "code",
1164
          "expr": "sum(rate(workqueue_queue_duration_seconds_bucket{job=\"ws-manager-mk2\", pod=~\"$pod\", name=~\"$controller\"}[$__rate_interval])) by (le)",
1165
          "format": "heatmap",
1166
          "intervalFactor": 2,
1167
          "legendFormat": "__auto",
1168
          "range": true,
1169
          "refId": "A"
1170
        }
1171
      ],
1172
      "title": "Work Queue Latency",
1173
      "type": "heatmap"
1174
    },
1175
    {
1176
      "aliasColors": {},
1177
      "bars": false,
1178
      "dashLength": 10,
1179
      "dashes": false,
1180
      "datasource": {
1181
        "type": "prometheus",
1182
        "uid": "$datasource"
1183
      },
1184
      "fieldConfig": {
1185
        "defaults": {
1186
          "links": []
1187
        },
1188
        "overrides": []
1189
      },
1190
      "fill": 1,
1191
      "fillGradient": 0,
1192
      "gridPos": {
1193
        "h": 9,
1194
        "w": 8,
1195
        "x": 0,
1196
        "y": 31
1197
      },
1198
      "hiddenSeries": false,
1199
      "id": 136,
1200
      "legend": {
1201
        "alignAsTable": false,
1202
        "avg": false,
1203
        "current": false,
1204
        "max": false,
1205
        "min": false,
1206
        "rightSide": false,
1207
        "show": true,
1208
        "total": false,
1209
        "values": false
1210
      },
1211
      "lines": true,
1212
      "linewidth": 1,
1213
      "links": [],
1214
      "nullPointMode": "null",
1215
      "options": {
1216
        "alertThreshold": true
1217
      },
1218
      "percentage": false,
1219
      "pluginVersion": "9.5.2",
1220
      "pointradius": 5,
1221
      "points": false,
1222
      "renderer": "flot",
1223
      "seriesOverrides": [],
1224
      "spaceLength": 10,
1225
      "stack": false,
1226
      "steppedLine": false,
1227
      "targets": [
1228
        {
1229
          "datasource": {
1230
            "uid": "$datasource"
1231
          },
1232
          "editorMode": "code",
1233
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-manager-mk2\", pod=~\"$pod\",code=~\"2..\"}[5m]))",
1234
          "format": "time_series",
1235
          "intervalFactor": 2,
1236
          "legendFormat": "2xx",
1237
          "range": true,
1238
          "refId": "A"
1239
        },
1240
        {
1241
          "datasource": {
1242
            "uid": "$datasource"
1243
          },
1244
          "editorMode": "code",
1245
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-manager-mk2\", pod=~\"$pod\",code=~\"3..\"}[5m]))",
1246
          "format": "time_series",
1247
          "intervalFactor": 2,
1248
          "legendFormat": "3xx",
1249
          "range": true,
1250
          "refId": "B"
1251
        },
1252
        {
1253
          "datasource": {
1254
            "uid": "$datasource"
1255
          },
1256
          "editorMode": "code",
1257
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-manager-mk2\", pod=~\"$pod\",code=~\"4..\"}[5m]))",
1258
          "format": "time_series",
1259
          "intervalFactor": 2,
1260
          "legendFormat": "4xx",
1261
          "range": true,
1262
          "refId": "C"
1263
        },
1264
        {
1265
          "datasource": {
1266
            "uid": "$datasource"
1267
          },
1268
          "editorMode": "code",
1269
          "expr": "sum(rate(rest_client_requests_total{job=\"ws-manager-mk2\", pod=~\"$pod\",code=~\"5..\"}[5m]))",
1270
          "format": "time_series",
1271
          "intervalFactor": 2,
1272
          "legendFormat": "5xx",
1273
          "range": true,
1274
          "refId": "D"
1275
        }
1276
      ],
1277
      "thresholds": [],
1278
      "timeRegions": [],
1279
      "title": "Kube API Request Rate",
1280
      "tooltip": {
1281
        "shared": false,
1282
        "sort": 0,
1283
        "value_type": "individual"
1284
      },
1285
      "type": "graph",
1286
      "xaxis": {
1287
        "mode": "time",
1288
        "show": true,
1289
        "values": []
1290
      },
1291
      "yaxes": [
1292
        {
1293
          "$$hashKey": "object:705",
1294
          "format": "ops",
1295
          "logBase": 1,
1296
          "show": true
1297
        },
1298
        {
1299
          "$$hashKey": "object:706",
1300
          "format": "ops",
1301
          "logBase": 1,
1302
          "show": true
1303
        }
1304
      ],
1305
      "yaxis": {
1306
        "align": false
1307
      }
1308
    },
1309
    {
1310
      "datasource": {
1311
        "type": "prometheus",
1312
        "uid": "$datasource"
1313
      },
1314
      "fieldConfig": {
1315
        "defaults": {
1316
          "color": {
1317
            "mode": "palette-classic"
1318
          },
1319
          "custom": {
1320
            "axisCenteredZero": false,
1321
            "axisColorMode": "text",
1322
            "axisLabel": "",
1323
            "axisPlacement": "auto",
1324
            "barAlignment": 0,
1325
            "drawStyle": "line",
1326
            "fillOpacity": 10,
1327
            "gradientMode": "none",
1328
            "hideFrom": {
1329
              "legend": false,
1330
              "tooltip": false,
1331
              "viz": false
1332
            },
1333
            "lineInterpolation": "linear",
1334
            "lineWidth": 1,
1335
            "pointSize": 5,
1336
            "scaleDistribution": {
1337
              "type": "linear"
1338
            },
1339
            "showPoints": "never",
1340
            "spanNulls": false,
1341
            "stacking": {
1342
              "group": "A",
1343
              "mode": "none"
1344
            },
1345
            "thresholdsStyle": {
1346
              "mode": "off"
1347
            }
1348
          },
1349
          "links": [],
1350
          "mappings": [],
1351
          "min": 0,
1352
          "thresholds": {
1353
            "mode": "absolute",
1354
            "steps": [
1355
              {
1356
                "color": "green",
1357
                "value": null
1358
              },
1359
              {
1360
                "color": "red",
1361
                "value": 80
1362
              }
1363
            ]
1364
          },
1365
          "unit": "short"
1366
        },
1367
        "overrides": [
1368
          {
1369
            "matcher": {
1370
              "id": "byRegexp",
1371
              "options": ".*error"
1372
            },
1373
            "properties": [
1374
              {
1375
                "id": "color",
1376
                "value": {
1377
                  "fixedColor": "red",
1378
                  "mode": "fixed"
1379
                }
1380
              }
1381
            ]
1382
          },
1383
          {
1384
            "matcher": {
1385
              "id": "byRegexp",
1386
              "options": ".*requeue.*"
1387
            },
1388
            "properties": [
1389
              {
1390
                "id": "color",
1391
                "value": {
1392
                  "fixedColor": "blue",
1393
                  "mode": "fixed"
1394
                }
1395
              }
1396
            ]
1397
          },
1398
          {
1399
            "matcher": {
1400
              "id": "byRegexp",
1401
              "options": ".*success"
1402
            },
1403
            "properties": [
1404
              {
1405
                "id": "color",
1406
                "value": {
1407
                  "fixedColor": "green",
1408
                  "mode": "fixed"
1409
                }
1410
              }
1411
            ]
1412
          }
1413
        ]
1414
      },
1415
      "gridPos": {
1416
        "h": 9,
1417
        "w": 8,
1418
        "x": 8,
1419
        "y": 31
1420
      },
1421
      "id": 112,
1422
      "links": [],
1423
      "options": {
1424
        "legend": {
1425
          "calcs": [
1426
            "lastNotNull"
1427
          ],
1428
          "displayMode": "table",
1429
          "placement": "bottom",
1430
          "showLegend": true
1431
        },
1432
        "tooltip": {
1433
          "mode": "single",
1434
          "sort": "none"
1435
        }
1436
      },
1437
      "pluginVersion": "9.3.6",
1438
      "targets": [
1439
        {
1440
          "datasource": {
1441
            "uid": "$datasource"
1442
          },
1443
          "editorMode": "code",
1444
          "expr": "sum(rate(controller_runtime_reconcile_total{job=\"ws-manager-mk2\", pod=~\"$pod\", controller=~\"$controller\"}[5m])) by (cluster, pod, controller, result)",
1445
          "format": "time_series",
1446
          "intervalFactor": 2,
1447
          "legendFormat": "{{cluster}} {{pod}} {{controller}} {{result}}",
1448
          "range": true,
1449
          "refId": "A"
1450
        }
1451
      ],
1452
      "title": "Reconciliations",
1453
      "type": "timeseries"
1454
    },
1455
    {
1456
      "datasource": {
1457
        "type": "prometheus",
1458
        "uid": "$datasource"
1459
      },
1460
      "fieldConfig": {
1461
        "defaults": {
1462
          "custom": {
1463
            "hideFrom": {
1464
              "legend": false,
1465
              "tooltip": false,
1466
              "viz": false
1467
            },
1468
            "scaleDistribution": {
1469
              "type": "linear"
1470
            }
1471
          },
1472
          "links": []
1473
        },
1474
        "overrides": []
1475
      },
1476
      "gridPos": {
1477
        "h": 9,
1478
        "w": 8,
1479
        "x": 16,
1480
        "y": 31
1481
      },
1482
      "id": 158,
1483
      "links": [],
1484
      "options": {
1485
        "calculate": false,
1486
        "cellGap": 1,
1487
        "color": {
1488
          "exponent": 0.5,
1489
          "fill": "dark-orange",
1490
          "mode": "scheme",
1491
          "reverse": false,
1492
          "scale": "exponential",
1493
          "scheme": "Spectral",
1494
          "steps": 64
1495
        },
1496
        "exemplars": {
1497
          "color": "rgba(255,0,255,0.7)"
1498
        },
1499
        "filterValues": {
1500
          "le": 1e-9
1501
        },
1502
        "legend": {
1503
          "show": true
1504
        },
1505
        "rowsFrame": {
1506
          "layout": "auto"
1507
        },
1508
        "tooltip": {
1509
          "show": true,
1510
          "yHistogram": false
1511
        },
1512
        "yAxis": {
1513
          "axisPlacement": "left",
1514
          "reverse": false,
1515
          "unit": "s"
1516
        }
1517
      },
1518
      "pluginVersion": "9.5.2",
1519
      "targets": [
1520
        {
1521
          "datasource": {
1522
            "type": "prometheus",
1523
            "uid": "$datasource"
1524
          },
1525
          "editorMode": "code",
1526
          "expr": "sum(rate(workqueue_work_duration_seconds_bucket{job=\"ws-manager-mk2\", pod=~\"$pod\", name=~\"$controller\"}[$__rate_interval])) by (le)",
1527
          "format": "heatmap",
1528
          "hide": false,
1529
          "legendFormat": "__auto",
1530
          "range": true,
1531
          "refId": "B"
1532
        }
1533
      ],
1534
      "title": "Reconcile Work Duration",
1535
      "type": "heatmap"
1536
    },
1537
    {
1538
      "datasource": {
1539
        "type": "prometheus",
1540
        "uid": "$datasource"
1541
      },
1542
      "fieldConfig": {
1543
        "defaults": {
1544
          "color": {
1545
            "mode": "palette-classic"
1546
          },
1547
          "custom": {
1548
            "axisCenteredZero": false,
1549
            "axisColorMode": "text",
1550
            "axisLabel": "",
1551
            "axisPlacement": "auto",
1552
            "barAlignment": 0,
1553
            "drawStyle": "line",
1554
            "fillOpacity": 10,
1555
            "gradientMode": "none",
1556
            "hideFrom": {
1557
              "legend": false,
1558
              "tooltip": false,
1559
              "viz": false
1560
            },
1561
            "lineInterpolation": "linear",
1562
            "lineWidth": 1,
1563
            "pointSize": 5,
1564
            "scaleDistribution": {
1565
              "type": "linear"
1566
            },
1567
            "showPoints": "never",
1568
            "spanNulls": false,
1569
            "stacking": {
1570
              "group": "A",
1571
              "mode": "none"
1572
            },
1573
            "thresholdsStyle": {
1574
              "mode": "off"
1575
            }
1576
          },
1577
          "links": [],
1578
          "mappings": [],
1579
          "min": 0,
1580
          "thresholds": {
1581
            "mode": "absolute",
1582
            "steps": [
1583
              {
1584
                "color": "green",
1585
                "value": null
1586
              },
1587
              {
1588
                "color": "red",
1589
                "value": 80
1590
              }
1591
            ]
1592
          },
1593
          "unit": "s"
1594
        },
1595
        "overrides": [
1596
          {
1597
            "matcher": {
1598
              "id": "byRegexp",
1599
              "options": ".*error"
1600
            },
1601
            "properties": [
1602
              {
1603
                "id": "color",
1604
                "value": {
1605
                  "fixedColor": "red",
1606
                  "mode": "fixed"
1607
                }
1608
              }
1609
            ]
1610
          },
1611
          {
1612
            "matcher": {
1613
              "id": "byRegexp",
1614
              "options": ".*requeue.*"
1615
            },
1616
            "properties": [
1617
              {
1618
                "id": "color",
1619
                "value": {
1620
                  "fixedColor": "blue",
1621
                  "mode": "fixed"
1622
                }
1623
              }
1624
            ]
1625
          },
1626
          {
1627
            "matcher": {
1628
              "id": "byRegexp",
1629
              "options": ".*success"
1630
            },
1631
            "properties": [
1632
              {
1633
                "id": "color",
1634
                "value": {
1635
                  "fixedColor": "green",
1636
                  "mode": "fixed"
1637
                }
1638
              }
1639
            ]
1640
          }
1641
        ]
1642
      },
1643
      "gridPos": {
1644
        "h": 9,
1645
        "w": 8,
1646
        "x": 0,
1647
        "y": 40
1648
      },
1649
      "id": 183,
1650
      "links": [],
1651
      "options": {
1652
        "legend": {
1653
          "calcs": [
1654
            "lastNotNull"
1655
          ],
1656
          "displayMode": "table",
1657
          "placement": "bottom",
1658
          "showLegend": true
1659
        },
1660
        "tooltip": {
1661
          "mode": "single",
1662
          "sort": "none"
1663
        }
1664
      },
1665
      "pluginVersion": "9.3.6",
1666
      "targets": [
1667
        {
1668
          "datasource": {
1669
            "type": "prometheus",
1670
            "uid": "$datasource"
1671
          },
1672
          "editorMode": "code",
1673
          "expr": "topk(20, max(workqueue_longest_running_processor_seconds{job=\"ws-manager-mk2\", pod=~\"$pod\", name=~\"$controller\"}) by (cluster, pod, name))",
1674
          "hide": false,
1675
          "legendFormat": "{{cluster}} - {{pod}} - {{name}}",
1676
          "range": true,
1677
          "refId": "B"
1678
        }
1679
      ],
1680
      "title": "Longest running reconciliation",
1681
      "type": "timeseries"
1682
    },
1683
    {
1684
      "collapsed": false,
1685
      "datasource": {
1686
        "type": "datasource",
1687
        "uid": "grafana"
1688
      },
1689
      "gridPos": {
1690
        "h": 1,
1691
        "w": 24,
1692
        "x": 0,
1693
        "y": 49
1694
      },
1695
      "id": 65,
1696
      "panels": [],
1697
      "targets": [
1698
        {
1699
          "datasource": {
1700
            "type": "datasource",
1701
            "uid": "grafana"
1702
          },
1703
          "refId": "A"
1704
        }
1705
      ],
1706
      "title": "gRPC Metrics",
1707
      "type": "row"
1708
    },
1709
    {
1710
      "aliasColors": {},
1711
      "bars": false,
1712
      "dashLength": 10,
1713
      "dashes": false,
1714
      "datasource": {
1715
        "type": "prometheus",
1716
        "uid": "$datasource"
1717
      },
1718
      "description": "",
1719
      "fieldConfig": {
1720
        "defaults": {
1721
          "links": []
1722
        },
1723
        "overrides": []
1724
      },
1725
      "fill": 1,
1726
      "fillGradient": 0,
1727
      "gridPos": {
1728
        "h": 9,
1729
        "w": 12,
1730
        "x": 0,
1731
        "y": 50
1732
      },
1733
      "hiddenSeries": false,
1734
      "id": 66,
1735
      "legend": {
1736
        "alignAsTable": true,
1737
        "avg": false,
1738
        "current": true,
1739
        "max": true,
1740
        "min": false,
1741
        "show": true,
1742
        "total": false,
1743
        "values": true
1744
      },
1745
      "lines": true,
1746
      "linewidth": 1,
1747
      "links": [],
1748
      "nullPointMode": "null",
1749
      "options": {
1750
        "alertThreshold": true
1751
      },
1752
      "percentage": false,
1753
      "pluginVersion": "9.5.2",
1754
      "pointradius": 2,
1755
      "points": false,
1756
      "renderer": "flot",
1757
      "repeatDirection": "h",
1758
      "seriesOverrides": [],
1759
      "spaceLength": 10,
1760
      "stack": false,
1761
      "steppedLine": false,
1762
      "targets": [
1763
        {
1764
          "datasource": {
1765
            "uid": "$datasource"
1766
          },
1767
          "editorMode": "code",
1768
          "expr": "sum(rate(grpc_server_handled_total{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[1m])) by (grpc_code, grpc_method, cluster)",
1769
          "interval": "",
1770
          "legendFormat": "{{cluster}} - {{grpc_method}} - {{grpc_code}}",
1771
          "queryType": "randomWalk",
1772
          "range": true,
1773
          "refId": "A"
1774
        }
1775
      ],
1776
      "thresholds": [],
1777
      "timeRegions": [],
1778
      "title": "gRPC calls handled (Server-side)",
1779
      "tooltip": {
1780
        "shared": true,
1781
        "sort": 2,
1782
        "value_type": "individual"
1783
      },
1784
      "type": "graph",
1785
      "xaxis": {
1786
        "mode": "time",
1787
        "show": true,
1788
        "values": []
1789
      },
1790
      "yaxes": [
1791
        {
1792
          "$$hashKey": "object:434",
1793
          "decimals": 2,
1794
          "format": "reqps",
1795
          "logBase": 1,
1796
          "min": "0",
1797
          "show": true
1798
        },
1799
        {
1800
          "$$hashKey": "object:435",
1801
          "format": "reqps",
1802
          "logBase": 1,
1803
          "show": true
1804
        }
1805
      ],
1806
      "yaxis": {
1807
        "align": false
1808
      }
1809
    },
1810
    {
1811
      "aliasColors": {},
1812
      "bars": false,
1813
      "dashLength": 10,
1814
      "dashes": false,
1815
      "datasource": {
1816
        "type": "prometheus",
1817
        "uid": "$datasource"
1818
      },
1819
      "description": "",
1820
      "fieldConfig": {
1821
        "defaults": {
1822
          "links": []
1823
        },
1824
        "overrides": []
1825
      },
1826
      "fill": 1,
1827
      "fillGradient": 0,
1828
      "gridPos": {
1829
        "h": 9,
1830
        "w": 12,
1831
        "x": 12,
1832
        "y": 50
1833
      },
1834
      "hiddenSeries": false,
1835
      "id": 67,
1836
      "legend": {
1837
        "alignAsTable": true,
1838
        "avg": false,
1839
        "current": true,
1840
        "max": true,
1841
        "min": false,
1842
        "show": true,
1843
        "total": false,
1844
        "values": true
1845
      },
1846
      "lines": true,
1847
      "linewidth": 1,
1848
      "links": [],
1849
      "nullPointMode": "null",
1850
      "options": {
1851
        "alertThreshold": true
1852
      },
1853
      "percentage": false,
1854
      "pluginVersion": "9.5.2",
1855
      "pointradius": 2,
1856
      "points": false,
1857
      "renderer": "flot",
1858
      "repeatDirection": "h",
1859
      "seriesOverrides": [],
1860
      "spaceLength": 10,
1861
      "stack": false,
1862
      "steppedLine": false,
1863
      "targets": [
1864
        {
1865
          "datasource": {
1866
            "uid": "$datasource"
1867
          },
1868
          "editorMode": "code",
1869
          "expr": "sum(rate(grpc_server_started_total{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[1m])) by (grpc_code, grpc_method, cluster)",
1870
          "interval": "",
1871
          "legendFormat": "{{cluster}} - {{grpc_method}} - Started",
1872
          "queryType": "randomWalk",
1873
          "range": true,
1874
          "refId": "A"
1875
        },
1876
        {
1877
          "datasource": {
1878
            "uid": "$datasource"
1879
          },
1880
          "editorMode": "code",
1881
          "expr": "sum(rate(grpc_server_handled_total{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[1m])) by (grpc_method, cluster)",
1882
          "hide": false,
1883
          "interval": "",
1884
          "legendFormat": "{{cluster}} - {{grpc_method}} - Finished",
1885
          "queryType": "randomWalk",
1886
          "range": true,
1887
          "refId": "B"
1888
        }
1889
      ],
1890
      "thresholds": [],
1891
      "timeRegions": [],
1892
      "title": "gRPC start and finish rate",
1893
      "tooltip": {
1894
        "shared": true,
1895
        "sort": 2,
1896
        "value_type": "individual"
1897
      },
1898
      "type": "graph",
1899
      "xaxis": {
1900
        "mode": "time",
1901
        "show": true,
1902
        "values": []
1903
      },
1904
      "yaxes": [
1905
        {
1906
          "$$hashKey": "object:493",
1907
          "decimals": 2,
1908
          "format": "reqps",
1909
          "logBase": 1,
1910
          "min": "0",
1911
          "show": true
1912
        },
1913
        {
1914
          "$$hashKey": "object:494",
1915
          "format": "none",
1916
          "logBase": 1,
1917
          "show": true
1918
        }
1919
      ],
1920
      "yaxis": {
1921
        "align": false
1922
      }
1923
    },
1924
    {
1925
      "aliasColors": {},
1926
      "bars": false,
1927
      "dashLength": 10,
1928
      "dashes": false,
1929
      "datasource": {
1930
        "type": "prometheus",
1931
        "uid": "$datasource"
1932
      },
1933
      "description": "",
1934
      "fieldConfig": {
1935
        "defaults": {
1936
          "links": []
1937
        },
1938
        "overrides": []
1939
      },
1940
      "fill": 1,
1941
      "fillGradient": 0,
1942
      "gridPos": {
1943
        "h": 9,
1944
        "w": 12,
1945
        "x": 0,
1946
        "y": 59
1947
      },
1948
      "hiddenSeries": false,
1949
      "id": 68,
1950
      "legend": {
1951
        "alignAsTable": true,
1952
        "avg": false,
1953
        "current": true,
1954
        "max": true,
1955
        "min": false,
1956
        "show": true,
1957
        "total": false,
1958
        "values": true
1959
      },
1960
      "lines": true,
1961
      "linewidth": 1,
1962
      "links": [],
1963
      "maxPerRow": 2,
1964
      "nullPointMode": "null",
1965
      "options": {
1966
        "alertThreshold": true
1967
      },
1968
      "percentage": false,
1969
      "pluginVersion": "9.5.2",
1970
      "pointradius": 2,
1971
      "points": false,
1972
      "renderer": "flot",
1973
      "repeat": "grpc_method",
1974
      "repeatDirection": "h",
1975
      "seriesOverrides": [],
1976
      "spaceLength": 10,
1977
      "stack": false,
1978
      "steppedLine": false,
1979
      "targets": [
1980
        {
1981
          "datasource": {
1982
            "uid": "$datasource"
1983
          },
1984
          "editorMode": "code",
1985
          "expr": "histogram_quantile(0.99, \n  sum(\n    rate(grpc_server_handling_seconds_bucket{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method, le)\n)",
1986
          "interval": "",
1987
          "legendFormat": "{{cluster}} - {{grpc_method}} - 99th Percentile",
1988
          "queryType": "randomWalk",
1989
          "range": true,
1990
          "refId": "A"
1991
        },
1992
        {
1993
          "datasource": {
1994
            "uid": "$datasource"
1995
          },
1996
          "editorMode": "code",
1997
          "expr": "histogram_quantile(0.95, \n  sum(\n    rate(grpc_server_handling_seconds_bucket{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method, le)\n)",
1998
          "interval": "",
1999
          "legendFormat": "{{cluster}} - {{grpc_method}} - 95th Percentile",
2000
          "queryType": "randomWalk",
2001
          "range": true,
2002
          "refId": "B"
2003
        },
2004
        {
2005
          "datasource": {
2006
            "uid": "$datasource"
2007
          },
2008
          "editorMode": "code",
2009
          "expr": "histogram_quantile(0.50, \n  sum(\n    rate(grpc_server_handling_seconds_bucket{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=~\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method, le)\n)",
2010
          "interval": "",
2011
          "legendFormat": "{{cluster}} - {{grpc_method}} - 50th Percentile",
2012
          "queryType": "randomWalk",
2013
          "range": true,
2014
          "refId": "C"
2015
        },
2016
        {
2017
          "datasource": {
2018
            "uid": "$datasource"
2019
          },
2020
          "editorMode": "code",
2021
          "expr": "  sum(\n    rate(grpc_server_handling_seconds_sum{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method)\n  /\n  sum(\n    rate(grpc_server_handling_seconds_count{job=\"ws-manager-mk2\", cluster=~\"$cluster\", grpc_method=\"$grpc_method\"}[5m])\n  ) by (cluster, grpc_method)",
2022
          "hide": false,
2023
          "interval": "",
2024
          "legendFormat": "{{cluster}} - {{grpc_method}} - avg",
2025
          "queryType": "randomWalk",
2026
          "range": true,
2027
          "refId": "D"
2028
        }
2029
      ],
2030
      "thresholds": [],
2031
      "timeRegions": [],
2032
      "title": "$grpc_method: Time handling gRPC calls",
2033
      "tooltip": {
2034
        "shared": true,
2035
        "sort": 0,
2036
        "value_type": "individual"
2037
      },
2038
      "type": "graph",
2039
      "xaxis": {
2040
        "mode": "time",
2041
        "show": true,
2042
        "values": []
2043
      },
2044
      "yaxes": [
2045
        {
2046
          "$$hashKey": "object:560",
2047
          "format": "s",
2048
          "logBase": 1,
2049
          "min": "0",
2050
          "show": true
2051
        },
2052
        {
2053
          "$$hashKey": "object:561",
2054
          "format": "reqps",
2055
          "logBase": 1,
2056
          "show": true
2057
        }
2058
      ],
2059
      "yaxis": {
2060
        "align": false
2061
      }
2062
    },
2063
    {
2064
      "collapsed": false,
2065
      "datasource": {
2066
        "type": "datasource",
2067
        "uid": "grafana"
2068
      },
2069
      "gridPos": {
2070
        "h": 1,
2071
        "w": 24,
2072
        "x": 0,
2073
        "y": 113
2074
      },
2075
      "id": 16,
2076
      "panels": [],
2077
      "targets": [
2078
        {
2079
          "datasource": {
2080
            "type": "datasource",
2081
            "uid": "grafana"
2082
          },
2083
          "refId": "A"
2084
        }
2085
      ],
2086
      "title": "Pod Metrics",
2087
      "type": "row"
2088
    },
2089
    {
2090
      "datasource": {
2091
        "type": "prometheus",
2092
        "uid": "$datasource"
2093
      },
2094
      "fieldConfig": {
2095
        "defaults": {
2096
          "color": {
2097
            "mode": "palette-classic"
2098
          },
2099
          "custom": {
2100
            "axisCenteredZero": false,
2101
            "axisColorMode": "text",
2102
            "axisLabel": "",
2103
            "axisPlacement": "auto",
2104
            "barAlignment": 0,
2105
            "drawStyle": "line",
2106
            "fillOpacity": 0,
2107
            "gradientMode": "none",
2108
            "hideFrom": {
2109
              "legend": false,
2110
              "tooltip": false,
2111
              "viz": false
2112
            },
2113
            "lineInterpolation": "linear",
2114
            "lineWidth": 1,
2115
            "pointSize": 5,
2116
            "scaleDistribution": {
2117
              "type": "linear"
2118
            },
2119
            "showPoints": "auto",
2120
            "spanNulls": false,
2121
            "stacking": {
2122
              "group": "A",
2123
              "mode": "none"
2124
            },
2125
            "thresholdsStyle": {
2126
              "mode": "off"
2127
            }
2128
          },
2129
          "mappings": [],
2130
          "thresholds": {
2131
            "mode": "absolute",
2132
            "steps": [
2133
              {
2134
                "color": "green"
2135
              },
2136
              {
2137
                "color": "red",
2138
                "value": 80
2139
              }
2140
            ]
2141
          }
2142
        },
2143
        "overrides": []
2144
      },
2145
      "gridPos": {
2146
        "h": 7,
2147
        "w": 10,
2148
        "x": 0,
2149
        "y": 114
2150
      },
2151
      "id": 76,
2152
      "options": {
2153
        "legend": {
2154
          "calcs": [],
2155
          "displayMode": "table",
2156
          "placement": "right",
2157
          "showLegend": true
2158
        },
2159
        "tooltip": {
2160
          "mode": "single",
2161
          "sort": "none"
2162
        }
2163
      },
2164
      "targets": [
2165
        {
2166
          "datasource": {
2167
            "uid": "$datasource"
2168
          },
2169
          "editorMode": "code",
2170
          "exemplar": true,
2171
          "expr": "kube_pod_container_info{cluster=~\"$cluster\", pod=~\"$pod\", image=~\".+\", container=\"ws-manager-mk2\"}",
2172
          "interval": "",
2173
          "legendFormat": "{{cluster}} - {{image}}",
2174
          "queryType": "randomWalk",
2175
          "range": true,
2176
          "refId": "A"
2177
        }
2178
      ],
2179
      "title": "Container image version",
2180
      "type": "timeseries"
2181
    },
2182
    {
2183
      "aliasColors": {},
2184
      "bars": false,
2185
      "dashLength": 10,
2186
      "dashes": false,
2187
      "datasource": {
2188
        "uid": "$datasource"
2189
      },
2190
      "decimals": 2,
2191
      "fill": 1,
2192
      "fillGradient": 0,
2193
      "gridPos": {
2194
        "h": 7,
2195
        "w": 7,
2196
        "x": 10,
2197
        "y": 114
2198
      },
2199
      "hiddenSeries": false,
2200
      "id": 2,
2201
      "legend": {
2202
        "alignAsTable": true,
2203
        "avg": true,
2204
        "current": true,
2205
        "max": true,
2206
        "min": false,
2207
        "show": true,
2208
        "sort": "current",
2209
        "sortDesc": true,
2210
        "total": false,
2211
        "values": true
2212
      },
2213
      "lines": true,
2214
      "linewidth": 1,
2215
      "nullPointMode": "null",
2216
      "options": {
2217
        "alertThreshold": true
2218
      },
2219
      "percentage": false,
2220
      "pluginVersion": "9.4.3",
2221
      "pointradius": 2,
2222
      "points": false,
2223
      "renderer": "flot",
2224
      "seriesOverrides": [],
2225
      "spaceLength": 10,
2226
      "stack": false,
2227
      "steppedLine": false,
2228
      "targets": [
2229
        {
2230
          "datasource": {
2231
            "uid": "$datasource"
2232
          },
2233
          "expr": "sum(\n  rate(container_cpu_usage_seconds_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2234
          "interval": "",
2235
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Cores being used",
2236
          "queryType": "randomWalk",
2237
          "refId": "A"
2238
        }
2239
      ],
2240
      "thresholds": [],
2241
      "timeRegions": [],
2242
      "title": "CPU Utilization",
2243
      "tooltip": {
2244
        "shared": true,
2245
        "sort": 2,
2246
        "value_type": "individual"
2247
      },
2248
      "type": "graph",
2249
      "xaxis": {
2250
        "mode": "time",
2251
        "show": true,
2252
        "values": []
2253
      },
2254
      "yaxes": [
2255
        {
2256
          "$$hashKey": "object:403",
2257
          "decimals": 2,
2258
          "format": "short",
2259
          "logBase": 1,
2260
          "min": "0",
2261
          "show": true
2262
        },
2263
        {
2264
          "$$hashKey": "object:404",
2265
          "format": "short",
2266
          "logBase": 1,
2267
          "show": true
2268
        }
2269
      ],
2270
      "yaxis": {
2271
        "align": false
2272
      }
2273
    },
2274
    {
2275
      "aliasColors": {},
2276
      "bars": false,
2277
      "dashLength": 10,
2278
      "dashes": false,
2279
      "datasource": {
2280
        "uid": "$datasource"
2281
      },
2282
      "description": "Saturation > 100% means that the container is requesting more than its limits.\n\nKubernetes will start to throttle CPU when that happens. That's a sign of degraded performance.\n\n'No Data' indicates that the pod has no CPU limits.",
2283
      "fill": 1,
2284
      "fillGradient": 0,
2285
      "gridPos": {
2286
        "h": 7,
2287
        "w": 7,
2288
        "x": 17,
2289
        "y": 114
2290
      },
2291
      "hiddenSeries": false,
2292
      "id": 4,
2293
      "legend": {
2294
        "avg": false,
2295
        "current": false,
2296
        "max": false,
2297
        "min": false,
2298
        "show": true,
2299
        "total": false,
2300
        "values": false
2301
      },
2302
      "lines": true,
2303
      "linewidth": 1,
2304
      "nullPointMode": "null",
2305
      "options": {
2306
        "alertThreshold": true
2307
      },
2308
      "percentage": false,
2309
      "pluginVersion": "9.4.3",
2310
      "pointradius": 2,
2311
      "points": false,
2312
      "renderer": "flot",
2313
      "seriesOverrides": [
2314
        {
2315
          "$$hashKey": "object:470",
2316
          "alias": "/.*CPU Throttles/",
2317
          "yaxis": 2
2318
        }
2319
      ],
2320
      "spaceLength": 10,
2321
      "stack": false,
2322
      "steppedLine": false,
2323
      "targets": [
2324
        {
2325
          "datasource": {
2326
            "uid": "$datasource"
2327
          },
2328
          "expr": "sum(\n  rate(container_cpu_usage_seconds_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)\n/\nsum(\n  kube_pod_container_resource_limits_cpu_cores{container!=\"POD\", cluster=\"$cluster\", pod=~\"$pod\"}\n) by (pod, cluster, node)\n",
2329
          "interval": "",
2330
          "legendFormat": "{{cluster}} - {{node} - {{pod}} - CPU Saturation",
2331
          "queryType": "randomWalk",
2332
          "refId": "A"
2333
        },
2334
        {
2335
          "datasource": {
2336
            "uid": "$datasource"
2337
          },
2338
          "expr": "sum(\nrate(container_cpu_cfs_throttled_seconds_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod)",
2339
          "interval": "",
2340
          "legendFormat": "{{pod}} - CPU Throttles",
2341
          "queryType": "randomWalk",
2342
          "refId": "B"
2343
        }
2344
      ],
2345
      "thresholds": [],
2346
      "timeRegions": [],
2347
      "title": "CPU Saturation",
2348
      "tooltip": {
2349
        "shared": true,
2350
        "sort": 2,
2351
        "value_type": "individual"
2352
      },
2353
      "type": "graph",
2354
      "xaxis": {
2355
        "mode": "time",
2356
        "show": true,
2357
        "values": []
2358
      },
2359
      "yaxes": [
2360
        {
2361
          "$$hashKey": "object:477",
2362
          "decimals": 2,
2363
          "format": "percentunit",
2364
          "logBase": 1,
2365
          "min": "0",
2366
          "show": true
2367
        },
2368
        {
2369
          "$$hashKey": "object:478",
2370
          "format": "s",
2371
          "logBase": 1,
2372
          "show": true
2373
        }
2374
      ],
2375
      "yaxis": {
2376
        "align": false
2377
      }
2378
    },
2379
    {
2380
      "aliasColors": {},
2381
      "bars": false,
2382
      "dashLength": 10,
2383
      "dashes": false,
2384
      "datasource": {
2385
        "uid": "$datasource"
2386
      },
2387
      "fill": 1,
2388
      "fillGradient": 0,
2389
      "gridPos": {
2390
        "h": 7,
2391
        "w": 12,
2392
        "x": 0,
2393
        "y": 121
2394
      },
2395
      "hiddenSeries": false,
2396
      "id": 6,
2397
      "legend": {
2398
        "alignAsTable": true,
2399
        "avg": true,
2400
        "current": true,
2401
        "max": true,
2402
        "min": false,
2403
        "show": true,
2404
        "total": false,
2405
        "values": true
2406
      },
2407
      "lines": true,
2408
      "linewidth": 1,
2409
      "nullPointMode": "null",
2410
      "options": {
2411
        "alertThreshold": true
2412
      },
2413
      "percentage": false,
2414
      "pluginVersion": "9.4.3",
2415
      "pointradius": 2,
2416
      "points": false,
2417
      "renderer": "flot",
2418
      "seriesOverrides": [],
2419
      "spaceLength": 10,
2420
      "stack": false,
2421
      "steppedLine": false,
2422
      "targets": [
2423
        {
2424
          "datasource": {
2425
            "uid": "$datasource"
2426
          },
2427
          "expr": "sum(container_memory_working_set_bytes{container!=\"POD\", container!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}) by (pod, cluster, node)",
2428
          "interval": "",
2429
          "legendFormat": "{{cluster}} - {{node}} - {{pod}}",
2430
          "queryType": "randomWalk",
2431
          "refId": "A"
2432
        }
2433
      ],
2434
      "thresholds": [],
2435
      "timeRegions": [],
2436
      "title": "Memory Utilization",
2437
      "tooltip": {
2438
        "shared": true,
2439
        "sort": 2,
2440
        "value_type": "individual"
2441
      },
2442
      "type": "graph",
2443
      "xaxis": {
2444
        "mode": "time",
2445
        "show": true,
2446
        "values": []
2447
      },
2448
      "yaxes": [
2449
        {
2450
          "$$hashKey": "object:536",
2451
          "format": "bytes",
2452
          "logBase": 1,
2453
          "min": "0",
2454
          "show": true
2455
        },
2456
        {
2457
          "$$hashKey": "object:537",
2458
          "format": "short",
2459
          "logBase": 1,
2460
          "show": true
2461
        }
2462
      ],
2463
      "yaxis": {
2464
        "align": false
2465
      }
2466
    },
2467
    {
2468
      "aliasColors": {},
2469
      "bars": false,
2470
      "dashLength": 10,
2471
      "dashes": false,
2472
      "datasource": {
2473
        "uid": "$datasource"
2474
      },
2475
      "decimals": 4,
2476
      "description": "Memory can't be throttled. When a container reaches 100% of its memory limits,  Kubernetes will kill the container and restart it.\n\n'No Data' indicates that the pod doesn't have Memory limits.",
2477
      "fill": 1,
2478
      "fillGradient": 0,
2479
      "gridPos": {
2480
        "h": 7,
2481
        "w": 12,
2482
        "x": 12,
2483
        "y": 121
2484
      },
2485
      "hiddenSeries": false,
2486
      "id": 8,
2487
      "legend": {
2488
        "alignAsTable": true,
2489
        "avg": true,
2490
        "current": true,
2491
        "max": true,
2492
        "min": false,
2493
        "show": true,
2494
        "total": false,
2495
        "values": true
2496
      },
2497
      "lines": true,
2498
      "linewidth": 1,
2499
      "nullPointMode": "null",
2500
      "options": {
2501
        "alertThreshold": true
2502
      },
2503
      "percentage": false,
2504
      "pluginVersion": "9.4.3",
2505
      "pointradius": 2,
2506
      "points": false,
2507
      "renderer": "flot",
2508
      "seriesOverrides": [],
2509
      "spaceLength": 10,
2510
      "stack": false,
2511
      "steppedLine": false,
2512
      "targets": [
2513
        {
2514
          "datasource": {
2515
            "uid": "$datasource"
2516
          },
2517
          "expr": "sum(\nrate(container_memory_working_set_bytes{container!=\"POD\", container!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)\n/\nsum(\n  kube_pod_container_resource_limits{container!=\"POD\", cluster=\"$cluster\", pod=~\"$pod\", resource=\"memory\"}\n) by (pod, cluster, node)\n",
2518
          "interval": "",
2519
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Memory Saturation",
2520
          "queryType": "randomWalk",
2521
          "refId": "A"
2522
        }
2523
      ],
2524
      "thresholds": [],
2525
      "timeRegions": [],
2526
      "title": "Memory Saturation",
2527
      "tooltip": {
2528
        "shared": true,
2529
        "sort": 2,
2530
        "value_type": "individual"
2531
      },
2532
      "type": "graph",
2533
      "xaxis": {
2534
        "mode": "time",
2535
        "show": true,
2536
        "values": []
2537
      },
2538
      "yaxes": [
2539
        {
2540
          "$$hashKey": "object:595",
2541
          "decimals": 2,
2542
          "format": "percentunit",
2543
          "logBase": 1,
2544
          "min": "0",
2545
          "show": true
2546
        },
2547
        {
2548
          "$$hashKey": "object:596",
2549
          "format": "short",
2550
          "logBase": 1,
2551
          "show": true
2552
        }
2553
      ],
2554
      "yaxis": {
2555
        "align": false
2556
      }
2557
    },
2558
    {
2559
      "aliasColors": {},
2560
      "bars": false,
2561
      "dashLength": 10,
2562
      "dashes": false,
2563
      "datasource": {
2564
        "uid": "$datasource"
2565
      },
2566
      "fill": 1,
2567
      "fillGradient": 0,
2568
      "gridPos": {
2569
        "h": 7,
2570
        "w": 8,
2571
        "x": 0,
2572
        "y": 128
2573
      },
2574
      "hiddenSeries": false,
2575
      "id": 10,
2576
      "legend": {
2577
        "alignAsTable": true,
2578
        "avg": false,
2579
        "current": true,
2580
        "max": false,
2581
        "min": false,
2582
        "show": true,
2583
        "total": false,
2584
        "values": true
2585
      },
2586
      "lines": true,
2587
      "linewidth": 1,
2588
      "nullPointMode": "null",
2589
      "options": {
2590
        "alertThreshold": true
2591
      },
2592
      "percentage": false,
2593
      "pluginVersion": "9.4.3",
2594
      "pointradius": 2,
2595
      "points": false,
2596
      "renderer": "flot",
2597
      "seriesOverrides": [],
2598
      "spaceLength": 10,
2599
      "stack": false,
2600
      "steppedLine": false,
2601
      "targets": [
2602
        {
2603
          "datasource": {
2604
            "uid": "$datasource"
2605
          },
2606
          "expr": "sum (\n  rate(container_network_receive_bytes_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2607
          "interval": "",
2608
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Received",
2609
          "queryType": "randomWalk",
2610
          "refId": "A"
2611
        },
2612
        {
2613
          "datasource": {
2614
            "uid": "$datasource"
2615
          },
2616
          "expr": "sum (\n  rate(container_network_transmit_bytes_total{container!=\"POD\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2617
          "interval": "",
2618
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Transmitted",
2619
          "queryType": "randomWalk",
2620
          "refId": "B"
2621
        }
2622
      ],
2623
      "thresholds": [],
2624
      "timeRegions": [],
2625
      "title": "Network Utilization",
2626
      "tooltip": {
2627
        "shared": true,
2628
        "sort": 2,
2629
        "value_type": "individual"
2630
      },
2631
      "type": "graph",
2632
      "xaxis": {
2633
        "mode": "time",
2634
        "show": true,
2635
        "values": []
2636
      },
2637
      "yaxes": [
2638
        {
2639
          "$$hashKey": "object:163",
2640
          "format": "binBps",
2641
          "logBase": 1,
2642
          "min": "0",
2643
          "show": true
2644
        },
2645
        {
2646
          "$$hashKey": "object:164",
2647
          "format": "short",
2648
          "logBase": 1,
2649
          "show": true
2650
        }
2651
      ],
2652
      "yaxis": {
2653
        "align": false
2654
      }
2655
    },
2656
    {
2657
      "aliasColors": {},
2658
      "bars": false,
2659
      "dashLength": 10,
2660
      "dashes": false,
2661
      "datasource": {
2662
        "uid": "$datasource"
2663
      },
2664
      "fill": 1,
2665
      "fillGradient": 0,
2666
      "gridPos": {
2667
        "h": 7,
2668
        "w": 8,
2669
        "x": 8,
2670
        "y": 128
2671
      },
2672
      "hiddenSeries": false,
2673
      "id": 57,
2674
      "legend": {
2675
        "alignAsTable": true,
2676
        "avg": false,
2677
        "current": true,
2678
        "max": false,
2679
        "min": false,
2680
        "rightSide": false,
2681
        "show": true,
2682
        "total": false,
2683
        "values": true
2684
      },
2685
      "lines": true,
2686
      "linewidth": 1,
2687
      "nullPointMode": "null",
2688
      "options": {
2689
        "alertThreshold": true
2690
      },
2691
      "percentage": false,
2692
      "pluginVersion": "9.4.3",
2693
      "pointradius": 2,
2694
      "points": false,
2695
      "renderer": "flot",
2696
      "seriesOverrides": [],
2697
      "spaceLength": 10,
2698
      "stack": false,
2699
      "steppedLine": false,
2700
      "targets": [
2701
        {
2702
          "datasource": {
2703
            "uid": "$datasource"
2704
          },
2705
          "expr": "sum (\n  rate(container_network_receive_packets_dropped_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2706
          "interval": "",
2707
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Receive",
2708
          "queryType": "randomWalk",
2709
          "refId": "A"
2710
        },
2711
        {
2712
          "datasource": {
2713
            "uid": "$datasource"
2714
          },
2715
          "expr": "sum (\n  rate(container_network_transmit_packets_dropped_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2716
          "interval": "",
2717
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Transmit",
2718
          "queryType": "randomWalk",
2719
          "refId": "B"
2720
        }
2721
      ],
2722
      "thresholds": [],
2723
      "timeRegions": [],
2724
      "title": "Network Saturation (Packets Dropped)",
2725
      "tooltip": {
2726
        "shared": true,
2727
        "sort": 2,
2728
        "value_type": "individual"
2729
      },
2730
      "type": "graph",
2731
      "xaxis": {
2732
        "mode": "time",
2733
        "show": true,
2734
        "values": []
2735
      },
2736
      "yaxes": [
2737
        {
2738
          "decimals": 2,
2739
          "format": "pps",
2740
          "logBase": 1,
2741
          "min": "0",
2742
          "show": true
2743
        },
2744
        {
2745
          "format": "short",
2746
          "logBase": 1,
2747
          "show": true
2748
        }
2749
      ],
2750
      "yaxis": {
2751
        "align": false
2752
      }
2753
    },
2754
    {
2755
      "aliasColors": {},
2756
      "bars": false,
2757
      "dashLength": 10,
2758
      "dashes": false,
2759
      "datasource": {
2760
        "uid": "$datasource"
2761
      },
2762
      "fill": 1,
2763
      "fillGradient": 0,
2764
      "gridPos": {
2765
        "h": 7,
2766
        "w": 8,
2767
        "x": 16,
2768
        "y": 128
2769
      },
2770
      "hiddenSeries": false,
2771
      "id": 59,
2772
      "legend": {
2773
        "alignAsTable": true,
2774
        "avg": false,
2775
        "current": true,
2776
        "max": false,
2777
        "min": false,
2778
        "show": true,
2779
        "total": false,
2780
        "values": true
2781
      },
2782
      "lines": true,
2783
      "linewidth": 1,
2784
      "nullPointMode": "null",
2785
      "options": {
2786
        "alertThreshold": true
2787
      },
2788
      "percentage": false,
2789
      "pluginVersion": "9.4.3",
2790
      "pointradius": 2,
2791
      "points": false,
2792
      "renderer": "flot",
2793
      "seriesOverrides": [],
2794
      "spaceLength": 10,
2795
      "stack": false,
2796
      "steppedLine": false,
2797
      "targets": [
2798
        {
2799
          "datasource": {
2800
            "uid": "$datasource"
2801
          },
2802
          "expr": "sum (\n  rate(container_network_receive_errors_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2803
          "interval": "",
2804
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Received",
2805
          "queryType": "randomWalk",
2806
          "refId": "A"
2807
        },
2808
        {
2809
          "datasource": {
2810
            "uid": "$datasource"
2811
          },
2812
          "expr": "sum (\n  rate(container_network_transmit_errors_total{container!=\"POD\", pod!=\"\", cluster=~\"$cluster\", node=~\"$node\", pod=~\"$pod\"}[1m])\n) by (pod, cluster, node)",
2813
          "interval": "",
2814
          "legendFormat": "{{cluster}} - {{node}} - {{pod}} - Transmitted",
2815
          "queryType": "randomWalk",
2816
          "refId": "B"
2817
        }
2818
      ],
2819
      "thresholds": [],
2820
      "timeRegions": [],
2821
      "title": "Network Errors",
2822
      "tooltip": {
2823
        "shared": true,
2824
        "sort": 2,
2825
        "value_type": "individual"
2826
      },
2827
      "type": "graph",
2828
      "xaxis": {
2829
        "mode": "time",
2830
        "show": true,
2831
        "values": []
2832
      },
2833
      "yaxes": [
2834
        {
2835
          "decimals": 2,
2836
          "format": "Errors/s",
2837
          "logBase": 1,
2838
          "min": "0",
2839
          "show": true
2840
        },
2841
        {
2842
          "format": "short",
2843
          "logBase": 1,
2844
          "show": true
2845
        }
2846
      ],
2847
      "yaxis": {
2848
        "align": false
2849
      }
2850
    },
2851
    {
2852
      "aliasColors": {},
2853
      "bars": false,
2854
      "dashLength": 10,
2855
      "dashes": false,
2856
      "datasource": {
2857
        "uid": "$datasource"
2858
      },
2859
      "decimals": 4,
2860
      "description": "",
2861
      "fill": 1,
2862
      "fillGradient": 0,
2863
      "gridPos": {
2864
        "h": 9,
2865
        "w": 8,
2866
        "x": 0,
2867
        "y": 135
2868
      },
2869
      "hiddenSeries": false,
2870
      "id": 36,
2871
      "legend": {
2872
        "alignAsTable": true,
2873
        "avg": true,
2874
        "current": true,
2875
        "max": true,
2876
        "min": false,
2877
        "show": true,
2878
        "total": false,
2879
        "values": true
2880
      },
2881
      "lines": true,
2882
      "linewidth": 1,
2883
      "nullPointMode": "null",
2884
      "options": {
2885
        "alertThreshold": true
2886
      },
2887
      "percentage": false,
2888
      "pluginVersion": "9.4.3",
2889
      "pointradius": 2,
2890
      "points": false,
2891
      "renderer": "flot",
2892
      "seriesOverrides": [],
2893
      "spaceLength": 10,
2894
      "stack": false,
2895
      "steppedLine": false,
2896
      "targets": [
2897
        {
2898
          "datasource": {
2899
            "uid": "$datasource"
2900
          },
2901
          "expr": "rate(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", pod=~\"$pod\"}[1m])",
2902
          "interval": "",
2903
          "legendFormat": "{{cluster}} - {{kubernetes_pod_node_name}} - {{pod}} ",
2904
          "queryType": "randomWalk",
2905
          "refId": "A"
2906
        }
2907
      ],
2908
      "thresholds": [],
2909
      "timeRegions": [],
2910
      "title": "Pod Restarts",
2911
      "tooltip": {
2912
        "shared": true,
2913
        "sort": 2,
2914
        "value_type": "individual"
2915
      },
2916
      "type": "graph",
2917
      "xaxis": {
2918
        "mode": "time",
2919
        "show": true,
2920
        "values": []
2921
      },
2922
      "yaxes": [
2923
        {
2924
          "$$hashKey": "object:654",
2925
          "decimals": 2,
2926
          "format": "short",
2927
          "logBase": 1,
2928
          "min": "0",
2929
          "show": true
2930
        },
2931
        {
2932
          "$$hashKey": "object:655",
2933
          "format": "short",
2934
          "logBase": 1,
2935
          "show": true
2936
        }
2937
      ],
2938
      "yaxis": {
2939
        "align": false
2940
      }
2941
    },
2942
    {
2943
      "aliasColors": {},
2944
      "bars": false,
2945
      "dashLength": 10,
2946
      "dashes": false,
2947
      "datasource": {
2948
        "uid": "$datasource"
2949
      },
2950
      "decimals": 0,
2951
      "description": "",
2952
      "fill": 1,
2953
      "fillGradient": 0,
2954
      "gridPos": {
2955
        "h": 9,
2956
        "w": 8,
2957
        "x": 8,
2958
        "y": 135
2959
      },
2960
      "hiddenSeries": false,
2961
      "id": 55,
2962
      "legend": {
2963
        "alignAsTable": true,
2964
        "avg": false,
2965
        "current": true,
2966
        "max": false,
2967
        "min": false,
2968
        "show": true,
2969
        "total": false,
2970
        "values": true
2971
      },
2972
      "lines": true,
2973
      "linewidth": 1,
2974
      "nullPointMode": "null",
2975
      "options": {
2976
        "alertThreshold": true
2977
      },
2978
      "percentage": false,
2979
      "pluginVersion": "9.4.3",
2980
      "pointradius": 2,
2981
      "points": false,
2982
      "renderer": "flot",
2983
      "seriesOverrides": [],
2984
      "spaceLength": 10,
2985
      "stack": false,
2986
      "steppedLine": false,
2987
      "targets": [
2988
        {
2989
          "datasource": {
2990
            "uid": "$datasource"
2991
          },
2992
          "expr": "kube_pod_container_status_running{cluster=~\"$cluster\", pod=~\"$pod\"} == 1 ",
2993
          "interval": "",
2994
          "legendFormat": "{{pod}}  - RUNNING",
2995
          "queryType": "randomWalk",
2996
          "refId": "A"
2997
        },
2998
        {
2999
          "datasource": {
3000
            "uid": "$datasource"
3001
          },
3002
          "expr": "(\n  sum by (pod) (kube_pod_container_status_terminated{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n) * on(pod) group_left(reason) (\n  sum by (pod, reason) (kube_pod_container_status_terminated_reason{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n)",
3003
          "interval": "",
3004
          "legendFormat": "{{pod}}  - TERMINATED -> {{reason}}",
3005
          "queryType": "randomWalk",
3006
          "refId": "B"
3007
        },
3008
        {
3009
          "datasource": {
3010
            "uid": "$datasource"
3011
          },
3012
          "expr": "(\n  sum by (pod) (kube_pod_container_status_waiting{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n) * on(pod) group_left(reason) (\n  sum by (pod, reason) (kube_pod_container_status_waiting_reason{cluster=~\"$cluster\", pod=~\"$pod\"}) == 1\n)",
3013
          "interval": "",
3014
          "legendFormat": "{{pod}}  - WAITING -> {{reason}}",
3015
          "queryType": "randomWalk",
3016
          "refId": "C"
3017
        }
3018
      ],
3019
      "thresholds": [],
3020
      "timeRegions": [],
3021
      "title": "Pod Status",
3022
      "tooltip": {
3023
        "shared": true,
3024
        "sort": 2,
3025
        "value_type": "individual"
3026
      },
3027
      "type": "graph",
3028
      "xaxis": {
3029
        "mode": "time",
3030
        "show": true,
3031
        "values": []
3032
      },
3033
      "yaxes": [
3034
        {
3035
          "$$hashKey": "object:709",
3036
          "decimals": 0,
3037
          "format": "short",
3038
          "logBase": 1,
3039
          "show": true
3040
        },
3041
        {
3042
          "$$hashKey": "object:710",
3043
          "format": "short",
3044
          "logBase": 1,
3045
          "show": true
3046
        }
3047
      ],
3048
      "yaxis": {
3049
        "align": false
3050
      }
3051
    },
3052
    {
3053
      "aliasColors": {},
3054
      "bars": false,
3055
      "dashLength": 10,
3056
      "dashes": false,
3057
      "datasource": {
3058
        "type": "prometheus",
3059
        "uid": "$datasource"
3060
      },
3061
      "decimals": 0,
3062
      "fill": 1,
3063
      "fillGradient": 0,
3064
      "gridPos": {
3065
        "h": 9,
3066
        "w": 8,
3067
        "x": 16,
3068
        "y": 135
3069
      },
3070
      "hiddenSeries": false,
3071
      "id": 40,
3072
      "legend": {
3073
        "alignAsTable": true,
3074
        "avg": false,
3075
        "current": true,
3076
        "max": false,
3077
        "min": false,
3078
        "show": true,
3079
        "total": false,
3080
        "values": true
3081
      },
3082
      "lines": true,
3083
      "linewidth": 1,
3084
      "nullPointMode": "null",
3085
      "options": {
3086
        "alertThreshold": true
3087
      },
3088
      "percentage": false,
3089
      "pluginVersion": "9.4.3",
3090
      "pointradius": 2,
3091
      "points": false,
3092
      "renderer": "flot",
3093
      "seriesOverrides": [],
3094
      "spaceLength": 10,
3095
      "stack": false,
3096
      "steppedLine": false,
3097
      "targets": [
3098
        {
3099
          "datasource": {
3100
            "uid": "$datasource"
3101
          },
3102
          "editorMode": "code",
3103
          "expr": "kube_deployment_spec_replicas{cluster=~\"$cluster\", deployment=\"ws-manager-mk2\"}",
3104
          "interval": "",
3105
          "legendFormat": "{{cluster}} - {{deployment}} - Desired",
3106
          "queryType": "randomWalk",
3107
          "range": true,
3108
          "refId": "C"
3109
        },
3110
        {
3111
          "datasource": {
3112
            "uid": "$datasource"
3113
          },
3114
          "editorMode": "code",
3115
          "expr": "kube_deployment_status_replicas_available{cluster=~\"$cluster\", deployment=\"ws-manager-mk2\"}",
3116
          "interval": "",
3117
          "legendFormat": "{{cluster}} - {{deployment}} - Available replicas",
3118
          "queryType": "randomWalk",
3119
          "range": true,
3120
          "refId": "A"
3121
        },
3122
        {
3123
          "datasource": {
3124
            "uid": "$datasource"
3125
          },
3126
          "editorMode": "code",
3127
          "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", deployment=\"ws-manager-mk2\"}",
3128
          "interval": "",
3129
          "legendFormat": "{{cluster}} - {{deployment}} - Unvailable replicas",
3130
          "queryType": "randomWalk",
3131
          "range": true,
3132
          "refId": "B"
3133
        }
3134
      ],
3135
      "thresholds": [],
3136
      "timeRegions": [],
3137
      "title": "Replicas availability",
3138
      "tooltip": {
3139
        "shared": true,
3140
        "sort": 2,
3141
        "value_type": "individual"
3142
      },
3143
      "type": "graph",
3144
      "xaxis": {
3145
        "mode": "time",
3146
        "show": true,
3147
        "values": []
3148
      },
3149
      "yaxes": [
3150
        {
3151
          "$$hashKey": "object:764",
3152
          "decimals": 0,
3153
          "format": "short",
3154
          "logBase": 1,
3155
          "show": true
3156
        },
3157
        {
3158
          "$$hashKey": "object:765",
3159
          "format": "short",
3160
          "logBase": 1,
3161
          "show": true
3162
        }
3163
      ],
3164
      "yaxis": {
3165
        "align": false
3166
      }
3167
    },
3168
    {
3169
      "collapsed": false,
3170
      "datasource": {
3171
        "type": "datasource",
3172
        "uid": "grafana"
3173
      },
3174
      "gridPos": {
3175
        "h": 1,
3176
        "w": 24,
3177
        "x": 0,
3178
        "y": 144
3179
      },
3180
      "id": 22,
3181
      "panels": [],
3182
      "targets": [
3183
        {
3184
          "datasource": {
3185
            "type": "datasource",
3186
            "uid": "grafana"
3187
          },
3188
          "refId": "A"
3189
        }
3190
      ],
3191
      "title": "Go Runtime Metrics",
3192
      "type": "row"
3193
    },
3194
    {
3195
      "aliasColors": {},
3196
      "bars": false,
3197
      "dashLength": 10,
3198
      "dashes": false,
3199
      "datasource": {
3200
        "type": "prometheus",
3201
        "uid": "$datasource"
3202
      },
3203
      "fill": 1,
3204
      "fillGradient": 0,
3205
      "gridPos": {
3206
        "h": 9,
3207
        "w": 12,
3208
        "x": 0,
3209
        "y": 145
3210
      },
3211
      "hiddenSeries": false,
3212
      "id": 32,
3213
      "legend": {
3214
        "alignAsTable": true,
3215
        "avg": true,
3216
        "current": true,
3217
        "max": true,
3218
        "min": false,
3219
        "show": true,
3220
        "total": false,
3221
        "values": true
3222
      },
3223
      "lines": true,
3224
      "linewidth": 1,
3225
      "nullPointMode": "null",
3226
      "options": {
3227
        "alertThreshold": true
3228
      },
3229
      "percentage": false,
3230
      "pluginVersion": "9.4.3",
3231
      "pointradius": 2,
3232
      "points": false,
3233
      "renderer": "flot",
3234
      "seriesOverrides": [],
3235
      "spaceLength": 10,
3236
      "stack": false,
3237
      "steppedLine": false,
3238
      "targets": [
3239
        {
3240
          "datasource": {
3241
            "uid": "$datasource"
3242
          },
3243
          "editorMode": "code",
3244
          "expr": "process_resident_memory_bytes{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\"}",
3245
          "interval": "",
3246
          "legendFormat": "{{cluster}} - {{pod}}",
3247
          "queryType": "randomWalk",
3248
          "range": true,
3249
          "refId": "A"
3250
        }
3251
      ],
3252
      "thresholds": [],
3253
      "timeRegions": [],
3254
      "title": "Memory Usage (as seen by the runtime process)",
3255
      "tooltip": {
3256
        "shared": true,
3257
        "sort": 0,
3258
        "value_type": "individual"
3259
      },
3260
      "type": "graph",
3261
      "xaxis": {
3262
        "mode": "time",
3263
        "show": true,
3264
        "values": []
3265
      },
3266
      "yaxes": [
3267
        {
3268
          "$$hashKey": "object:581",
3269
          "format": "bytes",
3270
          "logBase": 1,
3271
          "min": "0",
3272
          "show": true
3273
        },
3274
        {
3275
          "$$hashKey": "object:582",
3276
          "format": "short",
3277
          "logBase": 1,
3278
          "show": true
3279
        }
3280
      ],
3281
      "yaxis": {
3282
        "align": false
3283
      }
3284
    },
3285
    {
3286
      "aliasColors": {},
3287
      "bars": false,
3288
      "dashLength": 10,
3289
      "dashes": false,
3290
      "datasource": {
3291
        "type": "prometheus",
3292
        "uid": "$datasource"
3293
      },
3294
      "decimals": 2,
3295
      "editable": true,
3296
      "error": false,
3297
      "fill": 1,
3298
      "fillGradient": 0,
3299
      "gridPos": {
3300
        "h": 9,
3301
        "w": 12,
3302
        "x": 12,
3303
        "y": 145
3304
      },
3305
      "hiddenSeries": false,
3306
      "id": 34,
3307
      "legend": {
3308
        "alignAsTable": true,
3309
        "avg": true,
3310
        "current": true,
3311
        "hideEmpty": false,
3312
        "max": true,
3313
        "min": false,
3314
        "rightSide": false,
3315
        "show": true,
3316
        "total": false,
3317
        "values": true
3318
      },
3319
      "lines": true,
3320
      "linewidth": 1,
3321
      "links": [],
3322
      "nullPointMode": "null",
3323
      "options": {
3324
        "alertThreshold": true
3325
      },
3326
      "percentage": false,
3327
      "pluginVersion": "9.4.3",
3328
      "pointradius": 5,
3329
      "points": false,
3330
      "renderer": "flot",
3331
      "seriesOverrides": [],
3332
      "spaceLength": 10,
3333
      "stack": false,
3334
      "steppedLine": false,
3335
      "targets": [
3336
        {
3337
          "datasource": {
3338
            "uid": "$datasource"
3339
          },
3340
          "editorMode": "code",
3341
          "expr": "rate(process_cpu_seconds_total{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\"}[5m])",
3342
          "interval": "",
3343
          "intervalFactor": 2,
3344
          "legendFormat": "{{cluster}} - {{pod}}",
3345
          "metric": "prometheus_local_storage_ingested_samples_total",
3346
          "range": true,
3347
          "refId": "A",
3348
          "step": 10
3349
        }
3350
      ],
3351
      "thresholds": [],
3352
      "timeRegions": [],
3353
      "title": "CPU Usage (as seen by the runtime process)",
3354
      "tooltip": {
3355
        "msResolution": false,
3356
        "shared": true,
3357
        "sort": 2,
3358
        "value_type": "individual"
3359
      },
3360
      "type": "graph",
3361
      "xaxis": {
3362
        "mode": "time",
3363
        "show": true,
3364
        "values": [
3365
          "avg"
3366
        ]
3367
      },
3368
      "yaxes": [
3369
        {
3370
          "$$hashKey": "object:503",
3371
          "decimals": 2,
3372
          "format": "none",
3373
          "logBase": 1,
3374
          "min": "0",
3375
          "show": true
3376
        },
3377
        {
3378
          "$$hashKey": "object:504",
3379
          "format": "short",
3380
          "logBase": 1,
3381
          "show": true
3382
        }
3383
      ],
3384
      "yaxis": {
3385
        "align": false
3386
      }
3387
    },
3388
    {
3389
      "aliasColors": {},
3390
      "bars": false,
3391
      "dashLength": 10,
3392
      "dashes": false,
3393
      "datasource": {
3394
        "type": "prometheus",
3395
        "uid": "$datasource"
3396
      },
3397
      "fill": 1,
3398
      "fillGradient": 0,
3399
      "gridPos": {
3400
        "h": 9,
3401
        "w": 12,
3402
        "x": 0,
3403
        "y": 154
3404
      },
3405
      "hiddenSeries": false,
3406
      "id": 26,
3407
      "legend": {
3408
        "alignAsTable": true,
3409
        "avg": true,
3410
        "current": true,
3411
        "max": true,
3412
        "min": false,
3413
        "show": true,
3414
        "total": false,
3415
        "values": true
3416
      },
3417
      "lines": true,
3418
      "linewidth": 1,
3419
      "nullPointMode": "null",
3420
      "options": {
3421
        "alertThreshold": true
3422
      },
3423
      "percentage": false,
3424
      "pluginVersion": "9.4.3",
3425
      "pointradius": 2,
3426
      "points": false,
3427
      "renderer": "flot",
3428
      "seriesOverrides": [],
3429
      "spaceLength": 10,
3430
      "stack": false,
3431
      "steppedLine": false,
3432
      "targets": [
3433
        {
3434
          "datasource": {
3435
            "uid": "$datasource"
3436
          },
3437
          "editorMode": "code",
3438
          "expr": "go_memstats_heap_sys_bytes{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\"}",
3439
          "interval": "",
3440
          "legendFormat": "{{cluster}} - {{pod}}",
3441
          "queryType": "randomWalk",
3442
          "range": true,
3443
          "refId": "A"
3444
        }
3445
      ],
3446
      "thresholds": [],
3447
      "timeRegions": [],
3448
      "title": "Heap Usage",
3449
      "tooltip": {
3450
        "shared": true,
3451
        "sort": 2,
3452
        "value_type": "individual"
3453
      },
3454
      "type": "graph",
3455
      "xaxis": {
3456
        "mode": "time",
3457
        "show": true,
3458
        "values": []
3459
      },
3460
      "yaxes": [
3461
        {
3462
          "$$hashKey": "object:425",
3463
          "format": "bytes",
3464
          "logBase": 1,
3465
          "min": "0",
3466
          "show": true
3467
        },
3468
        {
3469
          "$$hashKey": "object:426",
3470
          "format": "short",
3471
          "logBase": 1,
3472
          "show": true
3473
        }
3474
      ],
3475
      "yaxis": {
3476
        "align": false
3477
      }
3478
    },
3479
    {
3480
      "aliasColors": {},
3481
      "bars": false,
3482
      "dashLength": 10,
3483
      "dashes": false,
3484
      "datasource": {
3485
        "type": "prometheus",
3486
        "uid": "$datasource"
3487
      },
3488
      "fieldConfig": {
3489
        "defaults": {
3490
          "links": []
3491
        },
3492
        "overrides": []
3493
      },
3494
      "fill": 1,
3495
      "fillGradient": 0,
3496
      "gridPos": {
3497
        "h": 9,
3498
        "w": 12,
3499
        "x": 12,
3500
        "y": 154
3501
      },
3502
      "hiddenSeries": false,
3503
      "id": 61,
3504
      "legend": {
3505
        "alignAsTable": true,
3506
        "avg": true,
3507
        "current": true,
3508
        "max": true,
3509
        "min": false,
3510
        "show": true,
3511
        "total": false,
3512
        "values": true
3513
      },
3514
      "lines": true,
3515
      "linewidth": 1,
3516
      "nullPointMode": "null",
3517
      "options": {
3518
        "alertThreshold": true
3519
      },
3520
      "percentage": false,
3521
      "pluginVersion": "9.4.3",
3522
      "pointradius": 2,
3523
      "points": false,
3524
      "renderer": "flot",
3525
      "repeatDirection": "h",
3526
      "seriesOverrides": [
3527
        {
3528
          "$$hashKey": "object:340",
3529
          "alias": "/GC rate:(.*)/",
3530
          "yaxis": 2
3531
        }
3532
      ],
3533
      "spaceLength": 10,
3534
      "stack": false,
3535
      "steppedLine": false,
3536
      "targets": [
3537
        {
3538
          "datasource": {
3539
            "uid": "$datasource"
3540
          },
3541
          "editorMode": "code",
3542
          "expr": "rate(go_memstats_alloc_bytes_total{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\"}[5m])",
3543
          "interval": "",
3544
          "legendFormat": "{{cluster}} - {{pod}}",
3545
          "range": true,
3546
          "refId": "A"
3547
        }
3548
      ],
3549
      "thresholds": [],
3550
      "timeRegions": [],
3551
      "title": "Memory Allocation rate",
3552
      "tooltip": {
3553
        "shared": true,
3554
        "sort": 2,
3555
        "value_type": "individual"
3556
      },
3557
      "type": "graph",
3558
      "xaxis": {
3559
        "mode": "time",
3560
        "show": true,
3561
        "values": []
3562
      },
3563
      "yaxes": [
3564
        {
3565
          "$$hashKey": "object:347",
3566
          "format": "binBps",
3567
          "logBase": 1,
3568
          "min": "0",
3569
          "show": true
3570
        },
3571
        {
3572
          "$$hashKey": "object:348",
3573
          "format": "short",
3574
          "logBase": 1,
3575
          "show": true
3576
        }
3577
      ],
3578
      "yaxis": {
3579
        "align": false
3580
      }
3581
    },
3582
    {
3583
      "aliasColors": {},
3584
      "bars": false,
3585
      "dashLength": 10,
3586
      "dashes": false,
3587
      "datasource": {
3588
        "type": "prometheus",
3589
        "uid": "$datasource"
3590
      },
3591
      "fill": 1,
3592
      "fillGradient": 0,
3593
      "gridPos": {
3594
        "h": 9,
3595
        "w": 12,
3596
        "x": 0,
3597
        "y": 163
3598
      },
3599
      "hiddenSeries": false,
3600
      "id": 24,
3601
      "legend": {
3602
        "alignAsTable": true,
3603
        "avg": true,
3604
        "current": true,
3605
        "max": true,
3606
        "min": false,
3607
        "show": true,
3608
        "total": false,
3609
        "values": true
3610
      },
3611
      "lines": true,
3612
      "linewidth": 1,
3613
      "nullPointMode": "null",
3614
      "options": {
3615
        "alertThreshold": true
3616
      },
3617
      "percentage": false,
3618
      "pluginVersion": "9.4.3",
3619
      "pointradius": 2,
3620
      "points": false,
3621
      "renderer": "flot",
3622
      "seriesOverrides": [],
3623
      "spaceLength": 10,
3624
      "stack": false,
3625
      "steppedLine": false,
3626
      "targets": [
3627
        {
3628
          "datasource": {
3629
            "uid": "$datasource"
3630
          },
3631
          "editorMode": "code",
3632
          "expr": "go_goroutines{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\"}",
3633
          "interval": "",
3634
          "legendFormat": "{{cluster}} - {{pod}}",
3635
          "queryType": "randomWalk",
3636
          "range": true,
3637
          "refId": "A"
3638
        }
3639
      ],
3640
      "thresholds": [],
3641
      "timeRegions": [],
3642
      "title": "Go Routines",
3643
      "tooltip": {
3644
        "shared": true,
3645
        "sort": 2,
3646
        "value_type": "individual"
3647
      },
3648
      "type": "graph",
3649
      "xaxis": {
3650
        "mode": "time",
3651
        "show": true,
3652
        "values": []
3653
      },
3654
      "yaxes": [
3655
        {
3656
          "$$hashKey": "object:184",
3657
          "format": "short",
3658
          "logBase": 1,
3659
          "min": "0",
3660
          "show": true
3661
        },
3662
        {
3663
          "$$hashKey": "object:185",
3664
          "format": "short",
3665
          "logBase": 1,
3666
          "show": true
3667
        }
3668
      ],
3669
      "yaxis": {
3670
        "align": false
3671
      }
3672
    },
3673
    {
3674
      "aliasColors": {},
3675
      "bars": false,
3676
      "dashLength": 10,
3677
      "dashes": false,
3678
      "datasource": {
3679
        "type": "prometheus",
3680
        "uid": "$datasource"
3681
      },
3682
      "fill": 1,
3683
      "fillGradient": 0,
3684
      "gridPos": {
3685
        "h": 9,
3686
        "w": 12,
3687
        "x": 12,
3688
        "y": 163
3689
      },
3690
      "hiddenSeries": false,
3691
      "id": 63,
3692
      "legend": {
3693
        "alignAsTable": true,
3694
        "avg": true,
3695
        "current": true,
3696
        "max": true,
3697
        "min": false,
3698
        "show": true,
3699
        "total": false,
3700
        "values": true
3701
      },
3702
      "lines": true,
3703
      "linewidth": 1,
3704
      "nullPointMode": "null",
3705
      "options": {
3706
        "alertThreshold": true
3707
      },
3708
      "percentage": false,
3709
      "pluginVersion": "9.4.3",
3710
      "pointradius": 2,
3711
      "points": false,
3712
      "renderer": "flot",
3713
      "seriesOverrides": [],
3714
      "spaceLength": 10,
3715
      "stack": false,
3716
      "steppedLine": false,
3717
      "targets": [
3718
        {
3719
          "datasource": {
3720
            "uid": "$datasource"
3721
          },
3722
          "editorMode": "code",
3723
          "expr": "go_gc_duration_seconds{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\", quantile=\"0.5\"}",
3724
          "interval": "",
3725
          "legendFormat": "{{cluster}} - {{pod}} - GC Duration - 50th percentile",
3726
          "range": true,
3727
          "refId": "A"
3728
        },
3729
        {
3730
          "datasource": {
3731
            "uid": "$datasource"
3732
          },
3733
          "editorMode": "code",
3734
          "expr": "go_gc_duration_seconds{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\", quantile=\"0.75\"}",
3735
          "interval": "",
3736
          "legendFormat": "{{cluster}} - {{pod}} - GC Duration - 75th percentile",
3737
          "range": true,
3738
          "refId": "B"
3739
        },
3740
        {
3741
          "datasource": {
3742
            "uid": "$datasource"
3743
          },
3744
          "editorMode": "code",
3745
          "expr": "go_gc_duration_seconds{cluster=~\"$cluster\", job=\"ws-manager-mk2\", pod=~\"$pod\", quantile=\"1\"}",
3746
          "interval": "",
3747
          "legendFormat": "{{cluster}} - {{pod}} - GC Duration - 100th percentile",
3748
          "range": true,
3749
          "refId": "C"
3750
        }
3751
      ],
3752
      "thresholds": [],
3753
      "timeRegions": [],
3754
      "title": "Garbage collection time",
3755
      "tooltip": {
3756
        "shared": true,
3757
        "sort": 2,
3758
        "value_type": "individual"
3759
      },
3760
      "type": "graph",
3761
      "xaxis": {
3762
        "mode": "time",
3763
        "show": true,
3764
        "values": []
3765
      },
3766
      "yaxes": [
3767
        {
3768
          "$$hashKey": "object:262",
3769
          "format": "s",
3770
          "logBase": 1,
3771
          "min": "0",
3772
          "show": true
3773
        },
3774
        {
3775
          "$$hashKey": "object:263",
3776
          "format": "short",
3777
          "logBase": 1,
3778
          "show": true
3779
        }
3780
      ],
3781
      "yaxis": {
3782
        "align": false
3783
      }
3784
    }
3785
  ],
3786
  "refresh": "30s",
3787
  "revision": 1,
3788
  "schemaVersion": 38,
3789
  "style": "dark",
3790
  "tags": [
3791
    "gitpod-mixin"
3792
  ],
3793
  "templating": {
3794
    "list": [
3795
      {
3796
        "current": {
3797
          "selected": true,
3798
          "text": [
3799
            "All"
3800
          ],
3801
          "value": [
3802
            "$__all"
3803
          ]
3804
        },
3805
        "datasource": {
3806
          "type": "prometheus",
3807
          "uid": "$datasource"
3808
        },
3809
        "definition": "label_values(up{job=\"ws-manager-mk2\"}, cluster)",
3810
        "hide": 0,
3811
        "includeAll": true,
3812
        "label": "Cluster",
3813
        "multi": true,
3814
        "name": "cluster",
3815
        "options": [],
3816
        "query": {
3817
          "query": "label_values(up{job=\"ws-manager-mk2\"}, cluster)",
3818
          "refId": "StandardVariableQuery"
3819
        },
3820
        "refresh": 2,
3821
        "regex": "",
3822
        "skipUrlSync": false,
3823
        "sort": 0,
3824
        "tagValuesQuery": "",
3825
        "tagsQuery": "",
3826
        "type": "query",
3827
        "useTags": false
3828
      },
3829
      {
3830
        "current": {
3831
          "selected": true,
3832
          "text": [
3833
            "All"
3834
          ],
3835
          "value": [
3836
            "$__all"
3837
          ]
3838
        },
3839
        "datasource": {
3840
          "type": "prometheus",
3841
          "uid": "$datasource"
3842
        },
3843
        "definition": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", pod=~\"ws-manager-mk2-.*\"}, node)",
3844
        "hide": 0,
3845
        "includeAll": true,
3846
        "label": "Node",
3847
        "multi": true,
3848
        "name": "node",
3849
        "options": [],
3850
        "query": {
3851
          "query": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", pod=~\"ws-manager-mk2-.*\"}, node)",
3852
          "refId": "StandardVariableQuery"
3853
        },
3854
        "refresh": 2,
3855
        "regex": "",
3856
        "skipUrlSync": false,
3857
        "sort": 0,
3858
        "tagValuesQuery": "",
3859
        "tagsQuery": "",
3860
        "type": "query",
3861
        "useTags": false
3862
      },
3863
      {
3864
        "current": {
3865
          "selected": true,
3866
          "text": [
3867
            "All"
3868
          ],
3869
          "value": [
3870
            "$__all"
3871
          ]
3872
        },
3873
        "datasource": {
3874
          "type": "prometheus",
3875
          "uid": "$datasource"
3876
        },
3877
        "definition": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", node=~\"$node\", pod=~\"ws-manager-mk2-.*\"}, pod)",
3878
        "hide": 0,
3879
        "includeAll": true,
3880
        "label": "Pod",
3881
        "multi": true,
3882
        "name": "pod",
3883
        "options": [],
3884
        "query": {
3885
          "query": "label_values(container_cpu_usage_seconds_total{cluster=~\"$cluster\", node=~\"$node\", pod=~\"ws-manager-mk2-.*\"}, pod)",
3886
          "refId": "StandardVariableQuery"
3887
        },
3888
        "refresh": 2,
3889
        "regex": "",
3890
        "skipUrlSync": false,
3891
        "sort": 0,
3892
        "tagValuesQuery": "",
3893
        "tagsQuery": "",
3894
        "type": "query",
3895
        "useTags": false
3896
      },
3897
      {
3898
        "current": {
3899
          "selected": true,
3900
          "text": [
3901
            "All"
3902
          ],
3903
          "value": [
3904
            "$__all"
3905
          ]
3906
        },
3907
        "datasource": {
3908
          "type": "prometheus",
3909
          "uid": "$datasource"
3910
        },
3911
        "definition": "label_values(grpc_server_handled_total{grpc_service=\"wsman.WorkspaceManager\", cluster=~\"$cluster\"}, grpc_method)",
3912
        "hide": 0,
3913
        "includeAll": true,
3914
        "label": "gRPC Method",
3915
        "multi": true,
3916
        "name": "grpc_method",
3917
        "options": [],
3918
        "query": {
3919
          "query": "label_values(grpc_server_handled_total{grpc_service=\"wsman.WorkspaceManager\", cluster=~\"$cluster\"}, grpc_method)",
3920
          "refId": "StandardVariableQuery"
3921
        },
3922
        "refresh": 2,
3923
        "regex": "",
3924
        "skipUrlSync": false,
3925
        "sort": 0,
3926
        "tagValuesQuery": "",
3927
        "tagsQuery": "",
3928
        "type": "query",
3929
        "useTags": false
3930
      },
3931
      {
3932
        "current": {
3933
          "selected": false,
3934
          "text": "VictoriaMetrics",
3935
          "value": "VictoriaMetrics"
3936
        },
3937
        "hide": 2,
3938
        "includeAll": false,
3939
        "multi": false,
3940
        "name": "datasource",
3941
        "options": [],
3942
        "query": "prometheus",
3943
        "refresh": 2,
3944
        "regex": "",
3945
        "skipUrlSync": false,
3946
        "type": "datasource"
3947
      },
3948
      {
3949
        "current": {
3950
          "selected": true,
3951
          "text": [
3952
            "All"
3953
          ],
3954
          "value": [
3955
            "$__all"
3956
          ]
3957
        },
3958
        "datasource": {
3959
          "type": "prometheus",
3960
          "uid": "${datasource}"
3961
        },
3962
        "definition": "label_values(workqueue_depth{pod=~\"$pod\"}, name)",
3963
        "hide": 0,
3964
        "includeAll": true,
3965
        "label": "Controller",
3966
        "multi": true,
3967
        "name": "controller",
3968
        "options": [],
3969
        "query": {
3970
          "query": "label_values(workqueue_depth{pod=~\"$pod\"}, name)",
3971
          "refId": "StandardVariableQuery"
3972
        },
3973
        "refresh": 1,
3974
        "regex": "",
3975
        "skipUrlSync": false,
3976
        "sort": 0,
3977
        "type": "query"
3978
      }
3979
    ]
3980
  },
3981
  "time": {
3982
    "from": "now-1h",
3983
    "to": "now"
3984
  },
3985
  "timepicker": {},
3986
  "timezone": "utc",
3987
  "title": "Gitpod / Component / ws-manager-mk2",
3988
  "uid": "ws-manager-mk2",
3989
  "version": 1,
3990
  "weekStart": ""
3991
}
3992

3993
Product

Resources

Company
