Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
gitpod-io
GitHub Repository: gitpod-io/gitpod
 Path: blob/main/operations/observability/mixins/self-hosted/rules/observability-stack/prometheus.yaml
3608 views
1
# Copyright (c) 2022 Gitpod GmbH. All rights reserved.

2
# Licensed under the GNU Affero General Public License (AGPL).

3
# See License.AGPL.txt in the project root for license information.

4


5
apiVersion: monitoring.coreos.com/v1

6
kind: PrometheusRule

7
metadata:

8
  labels:

9
    app.kubernetes.io/name: kubernetes

10
    app.kubernetes.io/part-of: kube-prometheus

11
    prometheus: k8s

12
    role: alert-rules

13
  name: prometheus-monitoring-rules

14
  namespace: monitoring-satellite

15
spec:

16
  groups:

17
  - name: prometheus

18
    rules:

19
    - alert: PrometheusBadConfig

20
      annotations:

21
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.

22
        summary: Failed Prometheus configuration reload.

23
      expr: |

24
        # Without max_over_time, failed scrapes could create false negatives, see

25
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.

26
        max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) == 0

27
      for: 10m

28
      labels:

29
        severity: critical

30
        team: delivery-operations-experience

31
    - alert: PrometheusRemoteStorageFailures

32
      annotations:

33
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}

34
        summary: Prometheus fails to send samples to remote storage.

35
      expr: |

36
        (

37
          (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]))

38
        /

39
          (

40
            (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]))

41
          +

42
            (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]))

43
          )

44
        )

45
        * 100

46
        > 1

47
      for: 15m

48
      labels:

49
        severity: critical

50
        team: delivery-operations-experience

51
    - alert: PrometheusRuleFailures

52
      annotations:

53
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.

54
        summary: Prometheus is failing rule evaluations.

55
      expr: |

56
        increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring-satellite"}[5m]) > 0

57
      for: 15m

58
      labels:

59
        severity: warning

60
        team: delivery-operations-experience

61
    - alert: PrometheusCrashlooped

62
      annotations:

63
        description: Prometheus' container restarted in the last 5m. While this alert will resolve itself if prometheus stopped crashing, it is important to understand why it crashed in the first place.

64
        summary: Prometheus has just crashlooped.

65
      expr: |

66
        increase(kube_pod_container_status_restarts_total{cluster=~"$cluster", pod="prometheus-k8s-0", container="prometheus"}[5m]) > 0

67
      for: 15m

68
      labels:

69
        severity: info

70
        team: delivery-operations-experience

71


72