Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
gitpod-io
GitHub Repository: gitpod-io/gitpod
 Path: blob/main/operations/observability/mixins/workspace/rules/central/workspacefailure-SLO.yaml
2506 views
1
# Copyright (c) 2022 Gitpod GmbH. All rights reserved.

2
# Licensed under the GNU Affero General Public License (AGPL).

3
# See License.AGPL.txt in the project root for license information.

4


5
apiVersion: monitoring.coreos.com/v1

6
kind: PrometheusRule

7
metadata:

8
  labels:

9
    prometheus: k8s

10
    role: alert-rules

11
  name: workspace-failure-slo-monitoring-rules

12
spec:

13
  groups:

14
  - name: workspace-failure-SLO

15
    rules:

16
    - record: gitpod_workspace_failure_total:5m_failure_ratio

17
      expr: |

18
        (

19
          (

20
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[5m]))

21
            /

22
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[5m]))

23
          )

24
        ) + (

25
          (

26
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[5m]))

27
            /

28
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[5m]))

29
          )

30
        )

31


32
    - record: gitpod_workspace_failure_total:30m_failure_ratio

33
      expr: |

34
        (

35
          (

36
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[30m]))

37
            /

38
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[30m]))

39
          )

40
        ) + (

41
          (

42
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[30m]))

43
            /

44
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[30m]))

45
          )

46
        )

47


48
    - record: gitpod_workspace_failure_total:1h_failure_ratio

49
      expr: |

50
        (

51
          (

52
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[1h]))

53
            /

54
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[1h]))

55
          )

56
        ) + (

57
          (

58
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[1h]))

59
            /

60
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[1h]))

61
          )

62
        )

63


64
    - record: gitpod_workspace_failure_total:2h_failure_ratio

65
      expr: |

66
        (

67
          (

68
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[2h]))

69
            /

70
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[2h]))

71
          )

72
        ) + (

73
          (

74
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[2h]))

75
            /

76
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[2h]))

77
          )

78
        )

79


80
    - record: gitpod_workspace_failure_total:6h_failure_ratio

81
      expr: |

82
        (

83
          (

84
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[6h]))

85
            /

86
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[6h]))

87
          )

88
        ) + (

89
          (

90
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[6h]))

91
            /

92
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[6h]))

93
          )

94
        )

95


96
    - record: gitpod_workspace_failure_total:1d_failure_ratio

97
      expr: |

98
        (

99
          (

100
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[1d]))

101
            /

102
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[1d]))

103
          )

104
        ) + (

105
          (

106
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[1d]))

107
            /

108
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[1d]))

109
          )

110
        )

111


112
    - record: gitpod_workspace_failure_total:3d_failure_ratio

113
      expr: |

114
        (

115
          (

116
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[3d]))

117
            /

118
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[3d]))

119
          )

120
        ) + (

121
          (

122
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[3d]))

123
            /

124
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[3d]))

125
          )

126
        )

127


128
    - record: gitpod_workspace_failure_total:30d_failure_ratio

129
      expr: |

130
        (

131
          (

132
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{reason="failed",type!~"Prebuild", cluster!~"ephemeral.*"}[30d]))

133
            /

134
            sum(rate(gitpod_ws_manager_mk2_workspace_stops_total{type!~"Prebuild", cluster!~"ephemeral.*"}[30d]))

135
          )

136
        ) + (

137
          (

138
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace",grpc_code!~"OK|ResourceExhausted", cluster!~"ephemeral.*"}[30d]))

139
            /

140
            sum(rate(grpc_server_handled_total{grpc_method="StartWorkspace", cluster!~"ephemeral.*"}[30d]))

141
          )

142
        )

143


144
    - record: gitpod_workspace_failure_total:slo_target

145
      expr: "0.99"

146
    - record: gitpod_workspace_failure_total:error_budget_remaining

147
      expr: gitpod_workspace_failure_total:monthly_availability - gitpod_workspace_failure_total:slo_target

148
    - record: gitpod_workspace_failure_total:monthly_availability

149
      expr: 1 - gitpod_workspace_failure_total:30d_failure_ratio

150


151