Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
gitpod-io
GitHub Repository: gitpod-io/gitpod
 Path: blob/main/operations/observability/mixins/workspace/rules/central/node-labeler.yaml
2506 views
1
# Copyright (c) 2022 Gitpod GmbH. All rights reserved.

2
# Licensed under the GNU Affero General Public License (AGPL).

3
# See License.AGPL.txt in the project root for license information.

4


5
apiVersion: monitoring.coreos.com/v1

6
kind: PrometheusRule

7
metadata:

8
  labels:

9
    prometheus: k8s

10
    role: alert-rules

11
  name: node-labeler-central-monitoring-rules

12
spec:

13
  groups:

14
    - name: node-labeler

15
      rules:

16
      - alert: GitpodNodeLabelerCrashLooping

17
        labels:

18
          severity: critical

19
          dedicated: included

20
        annotations:

21
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentCrashlooping.md

22
          summary: node-labeler is crashlooping in cluster {{ $labels.cluster }}.

23
          description: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 3 minutes.

24
        expr: |

25
          increase(kube_pod_container_status_restarts_total{container="node-labeler", cluster!~"ephemeral.*"}[1m]) > 3

26
        for: 3m

27
      - alert: GitpodNodeLabelerReplicasMismatch

28
        labels:

29
          severity: critical

30
          dedicated: included

31
        annotations:

32
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md

33
          summary: Desired number of replicas for node-labeler are not available in cluster {{ $labels.cluster }}

34
          description: 'Desired number of replicas for node-labeler are not available in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }} are missing'

35
        expr: |

36
          kube_deployment_spec_replicas{deployment="node-labeler", cluster!~"ephemeral.*"} != kube_deployment_status_replicas_available{deployment="node-labeler", cluster!~"ephemeral.*"}

37
        for: 3m

38
      - alert: GitpodNodeLabelerReplicaUnavailable

39
        labels:

40
          # TODO(gpl): warning for now, to set it up and fine-tune it

41
          severity: warning

42
        annotations:

43
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md

44
          summary: node-labeler replicas are unavailable in cluster {{ $labels.cluster }}

45
          description: 'node-labeler pods are unavailable in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }}'

46
        expr: |

47
          kube_deployment_status_replicas_unavailable{deployment="node-labeler", cluster!~"ephemeral.*"} > 0

48
        for: 10m

49


50