Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
gitpod-io
GitHub Repository: gitpod-io/gitpod
 Path: blob/main/operations/observability/mixins/workspace/rules/central/ipfs.yaml
2506 views
1
# Copyright (c) 2023 Gitpod GmbH. All rights reserved.

2
# Licensed under the GNU Affero General Public License (AGPL).

3
# See License.AGPL.txt in the project root for license information.

4


5
apiVersion: monitoring.coreos.com/v1

6
kind: PrometheusRule

7
metadata:

8
  labels:

9
    prometheus: k8s

10
    role: alert-rules

11
  name: ipfs-monitoring-rules

12
spec:

13
  groups:

14
    - name: ipfs-alerts

15
      rules:

16
        - alert: IPFSStorageLow

17
          labels:

18
            severity: warning

19
            team: engine

20
          for: 10m

21
          annotations:

22
            runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/IPFSAlmostOutOfStorage.md

23
            summary: IPFS is running out of storage within a few days in a workspace cluster. Create and shift to a new workspace cluster with a fresh IPFS.

24
            description: IPFS in cluster {{ $labels.cluster }} is running out of storage ({{ $value | humanize1024 }}B left). This happens naturally for workspace clusters over time, and is prevented by recreating the clusters regularly. Once out of storage, IPFS will stop working, and we fall back to the GCP registry for workspace image pulls, resulting in slower workspace startup times.

25
          expr: min(node_filesystem_avail_bytes{device="/dev/mapper/lvm--disk-ipfs", node=~"services-.*", cluster!~"ephemeral.*"}) by (cluster) < 600 * 1024*1024*1024

26
        - alert: IPFSStorageCritical

27
          labels:

28
            severity: critical

29
            team: engine

30
          for: 10m

31
          annotations:

32
            runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/IPFSAlmostOutOfStorage.md

33
            summary: IPFS is now very close to running out of storage in a workspace cluster. Create and shift to a new workspace cluster with a fresh IPFS.

34
            description: IPFS in cluster {{ $labels.cluster }} has almost ran out of storage ({{ $value | humanize1024 }}B left). This happens naturally for workspace clusters over time, and is prevented by recreating the clusters regularly. Once out of storage, IPFS will stop working, and we fall back to the GCP registry for workspace image pulls, resulting in slower workspace startup times.

35
          expr: min(node_filesystem_avail_bytes{device="/dev/mapper/lvm--disk-ipfs", node=~"services-.*", cluster!~"ephemeral.*"}) by (cluster) < 200 * 1024*1024*1024

36


37