Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
gitpod-io
GitHub Repository: gitpod-io/gitpod
 Path: blob/main/operations/observability/mixins/workspace/rules/central/image-builder.yaml
2506 views
1
# Copyright (c) 2022 Gitpod GmbH. All rights reserved.

2
# Licensed under the GNU Affero General Public License (AGPL).

3
# See License.AGPL.txt in the project root for license information.

4


5
apiVersion: monitoring.coreos.com/v1

6
kind: PrometheusRule

7
metadata:

8
  labels:

9
    prometheus: k8s

10
    role: alert-rules

11
  name: image-builder-central-monitoring-rules

12
spec:

13
  groups:

14
    - name: image-builder-central

15
      rules:

16
      - alert: GitpodImageBuildDurationAnomaly

17
        labels:

18
          severity: warning

19
          team: engine

20
        annotations:

21
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodImageBuildDurationAnomaly.md

22
          summary: image builds are happening too frequently in cluster {{ $labels.cluster }}

23
          description: Users are waiting more often for image builds

24
        expr: |

25
          (

26
              (

27
                  avg_over_time(gitpod_ws_manager_mk2_workspace_phase_total{phase="Running", type="ImageBuild", cluster!~"ephemeral.*"}[4h])

28
                  - avg_over_time(gitpod_ws_manager_mk2_workspace_phase_total{phase="Running", type="ImageBuild", cluster!~"ephemeral.*"}[7d])

29
              )

30
              / stddev_over_time(gitpod_ws_manager_mk2_workspace_phase_total{phase="Running", type="ImageBuild", cluster!~"ephemeral.*"}[30d])

31
          ) > 2.5

32
      - alert: GitpodImageBuilderCrashlooping

33
        labels:

34
          severity: critical

35
          dedicated: included

36
        annotations:

37
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentCrashlooping.md

38
          summary: image-builder-mk3 is crash looping in cluster {{ $labels.cluster }}

39
          description: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 3 minutes.

40
        expr: |

41
          increase(kube_pod_container_status_restarts_total{container="image-builder-mk3", cluster!~"ephemeral.*"}[1m]) > 3

42
        for: 3m

43
      - alert: GitpodImageBuilderReplicasMismatch

44
        labels:

45
          severity: critical

46
          dedicated: included

47
        annotations:

48
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md

49
          summary: Desired number of replicas for image-builder-mk3 are not available in cluster {{ $labels.cluster }}

50
          description: 'Desired number of replicas for image-builder-mk3 are not available in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }} are missing'

51
        expr: |

52
          kube_deployment_spec_replicas{deployment="image-builder-mk3", cluster!~"ephemeral.*"} != kube_deployment_status_replicas_available{deployment="image-builder-mk3", cluster!~"ephemeral.*"}

53
        for: 3m

54
      - alert: GitpodImageBuilderMk3ReplicaUnavailable

55
        labels:

56
          # TODO(gpl): warning for now, to set it up and fine-tune it

57
          severity: warning

58
        annotations:

59
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodWorkspaceDeploymentReplicaMismatch.md

60
          summary: image-builder-mk3 replicas are unavailable in cluster {{ $labels.cluster }}

61
          description: 'image-builder-mk3 pods are unavailable in cluster {{ $labels.cluster }}: {{ printf "%.2f" $value }}'

62
        expr: |

63
          kube_deployment_status_replicas_unavailable{deployment="image-builder-mk3", cluster!~"ephemeral.*"} > 0

64
        for: 10m

65
      - alert: GitpodImageBuilderMk3InternalErrors

66
        labels:

67
          severity: warning

68
          dedicated: included

69
          team: engine

70
        annotations:

71
          # runbook is 404 for now

72
          runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/GitpodImageBuilderMk3InternalErrors.md

73
          summary: image-builder-mk3 is returning unexpected internal errors

74
          description: 'Check the logs for image-builder-mk3 to inspect the high rate: {{ printf "%.2f" $value }}'

75
        expr: |

76
          sum by() (rate(grpc_server_handled_total{service="image-builder-mk3", grpc_code=~"Internal", grpc_method="Build"}[1h])) > 0.001

77
        for: 1h

78


79