Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
gitpod-io
GitHub Repository: gitpod-io/gitpod
 Path: blob/main/operations/observability/mixins/meta/rules/login-slo.yaml
3645 views
1
# Copyright (c) 2022 Gitpod GmbH. All rights reserved.

2
# Licensed under the GNU Affero General Public License (AGPL).

3
# See License.AGPL.txt in the project root for license information.

4


5
apiVersion: monitoring.coreos.com/v1

6
kind: PrometheusRule

7
metadata:

8
  labels:

9
    prometheus: k8s

10
    role: alert-rules

11
  name: login-slo-monitoring-rules

12
spec:

13
  groups:

14
  - name: login-slo-recording-rules

15
    rules:

16
    - record: gitpod_server_login_requests_total:5m_failure_ratio

17
      expr: |

18
        sum(rate(gitpod_server_login_requests_total{status="failed"}[5m]))

19
        /

20
        sum(rate(gitpod_server_login_requests_total[5m]))

21


22
    - record: gitpod_server_login_requests_total:30m_failure_ratio

23
      expr: |

24
        sum(rate(gitpod_server_login_requests_total{status="failed"}[30m]))

25
        /

26
        sum(rate(gitpod_server_login_requests_total[30m]))

27


28
    - record: gitpod_server_login_requests_total:1h_failure_ratio

29
      expr: |

30
        sum(rate(gitpod_server_login_requests_total{status="failed"}[1h]))

31
        /

32
        sum(rate(gitpod_server_login_requests_total[1h]))

33


34
    - record: gitpod_server_login_requests_total:2h_failure_ratio

35
      expr: |

36
        sum(rate(gitpod_server_login_requests_total{status="failed"}[2h]))

37
        /

38
        sum(rate(gitpod_server_login_requests_total[2h]))

39


40
    - record: gitpod_server_login_requests_total:6h_failure_ratio

41
      expr: |

42
        sum(rate(gitpod_server_login_requests_total{status="failed"}[6h]))

43
        /

44
        sum(rate(gitpod_server_login_requests_total[6h]))

45


46
    - record: gitpod_server_login_requests_total:1d_failure_ratio

47
      expr: |

48
        sum(rate(gitpod_server_login_requests_total{status="failed"}[1d]))

49
        /

50
        sum(rate(gitpod_server_login_requests_total[1d]))

51


52
    - record: gitpod_server_login_requests_total:3d_failure_ratio

53
      expr: |

54
        sum(rate(gitpod_server_login_requests_total{status="failed"}[3d]))

55
        /

56
        sum(rate(gitpod_server_login_requests_total[3d]))

57


58
    - record: gitpod_server_login_requests_total:30d_failure_ratio

59
      expr: |

60
        sum(rate(gitpod_server_login_requests_total{status="failed"}[30d]))

61
        /

62
        sum(rate(gitpod_server_login_requests_total[30d]))

63


64
    - record: gitpod_server_login_requests_total:slo_target

65
      expr: "0.95"

66


67
    - record: gitpod_server_login_requests_total:error_budget_remaining

68
      expr: gitpod_server_login_requests_total:monthly_availability - gitpod_server_login_requests_total:slo_target

69


70
    - record: gitpod_server_login_requests_total:monthly_availability

71
      expr: 1 - gitpod_server_login_requests_total:30d_failure_ratio

72


73
  - name: login-slo-alerts

74
    rules:

75
    - alert: GitpodLoginErrorBudgetBurn

76
      labels:

77
        severity: critical

78
      annotations:

79
        runbook_url: https://github.com/gitpod-com/observability/blob/main/runbooks/GitpodLoginErrorBudgetBurn.md

80
        summary: Error budget is being burn too quickly.

81
        description: Error budget is being burn too quickly. At this rate, the whole monthly budget will be burnt in less than 2 days.

82
      expr: |

83
        (

84
          gitpod_server_login_requests_total:1h_failure_ratio > (14.4 * (1 - gitpod_server_login_requests_total:slo_target))

85
          and

86
          gitpod_server_login_requests_total:5m_failure_ratio > (14.4 * (1 - gitpod_server_login_requests_total:slo_target))

87
        )

88
        or

89
        (

90
          gitpod_server_login_requests_total:6h_failure_ratio > (6 * (1 - gitpod_server_login_requests_total:slo_target))

91
          and

92
          gitpod_server_login_requests_total:30m_failure_ratio > (6 * (1 - gitpod_server_login_requests_total:slo_target))

93
        )

94


95
    - alert: GitpodLoginErrorBudgetBurn

96
      labels:

97
        severity: warning

98
      annotations:

99
        runbook_url: https://github.com/gitpod-com/observability/blob/main/runbooks/GitpodLoginErrorBudgetBurn.md

100
        summary: Error budget is being burn quickly.

101
        description: Error budget is being burn quickly. At this rate, the whole monthly budget will be burnt in less than 10 days.

102
      expr: |

103
        (

104
          gitpod_server_login_requests_total:1d_failure_ratio > (3 * (1 - gitpod_server_login_requests_total:slo_target))

105
          and

106
          gitpod_server_login_requests_total:2h_failure_ratio > (3 * (1 - gitpod_server_login_requests_total:slo_target))

107
        )

108
        or

109
        (

110
          gitpod_server_login_requests_total:3d_failure_ratio > (1 * (1 - gitpod_server_login_requests_total:slo_target))

111
          and

112
          gitpod_server_login_requests_total:6h_failure_ratio > (1 * (1 - gitpod_server_login_requests_total:slo_target))

113
        )

114


115