deployment.go

// Copyright (c) 2021 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License.AGPL.txt in the project root for license information.

package spicedb

import (
	"errors"
	"fmt"
	"strings"

	"github.com/gitpod-io/gitpod/common-go/baseserver"
	"github.com/gitpod-io/gitpod/installer/pkg/cluster"
	"github.com/gitpod-io/gitpod/installer/pkg/common"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/utils/pointer"
)

func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
	labels := common.CustomizeLabel(ctx, Component, common.TypeMetaDeployment)

	cfg := getExperimentalSpiceDBConfig(ctx)
	if cfg == nil || !cfg.Enabled {
		return nil, nil
	}

	if cfg.SecretRef == "" {
		return nil, errors.New("missing configuration for spicedb.secretRef")
	}

	bootstrapVolume, bootstrapVolumeMount, bootstrapFiles, contentHash, err := getBootstrapConfig(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get bootstrap config: %w", err)
	}

	replicas := common.Replicas(ctx, Component)

	return []runtime.Object{
		&appsv1.Deployment{
			TypeMeta: common.TypeMetaDeployment,
			ObjectMeta: metav1.ObjectMeta{
				Name:        Component,
				Namespace:   ctx.Namespace,
				Labels:      labels,
				Annotations: common.CustomizeAnnotation(ctx, Component, common.TypeMetaDeployment),
			},
			Spec: appsv1.DeploymentSpec{
				Selector: &metav1.LabelSelector{MatchLabels: common.DefaultLabels(Component)},
				Replicas: replicas,
				Strategy: common.DeploymentStrategy,
				Template: corev1.PodTemplateSpec{
					ObjectMeta: metav1.ObjectMeta{
						Name:      Component,
						Namespace: ctx.Namespace,
						Labels:    labels,
						Annotations: common.CustomizeAnnotation(ctx, Component, common.TypeMetaDeployment, func() map[string]string {
							return map[string]string{
								common.AnnotationConfigChecksum: contentHash,
							}
						}),
					},
					Spec: corev1.PodSpec{
						Affinity:                      cluster.WithNodeAffinityHostnameAntiAffinity(Component, cluster.AffinityLabelMeta),
						TopologySpreadConstraints:     cluster.WithHostnameTopologySpread(Component),
						PriorityClassName:             common.SystemNodeCritical,
						ServiceAccountName:            Component,
						EnableServiceLinks:            pointer.Bool(false),
						DNSPolicy:                     corev1.DNSClusterFirst,
						RestartPolicy:                 corev1.RestartPolicyAlways,
						TerminationGracePeriodSeconds: pointer.Int64(30),
						SecurityContext: &corev1.PodSecurityContext{
							RunAsNonRoot: pointer.Bool(false),
						},
						InitContainers: []corev1.Container{
							dbWaiter(ctx),
						},
						Containers: []corev1.Container{
							{
								Name:            ContainerName,
								Image:           ctx.ImageName(common.ThirdPartyContainerRepo(ctx.Config.Repository, RegistryRepo), RegistryImage, ImageTag),
								ImagePullPolicy: corev1.PullIfNotPresent,
								Args: (func() []string {
									args := []string{
										"serve",
										"--log-format=json",
										"--log-level=error",
										"--datastore-engine=mysql",
										"--datastore-conn-max-open=100",
										"--telemetry-endpoint=", // disable telemetry to https://telemetry.authzed.com
										fmt.Sprintf("--datastore-bootstrap-files=%s", strings.Join(bootstrapFiles, ",")),
										"--dispatch-cluster-enabled=true",
										"--datastore-bootstrap-overwrite=true",
										fmt.Sprintf("--metrics-addr=127.0.0.1:%d", baseserver.BuiltinMetricsPort),
									}

									// Dispatching only makes sense, when we have more than one replica
									if *replicas > 1 {
										args = append(args, fmt.Sprintf("--dispatch-upstream-addr=kubernetes:///spicedb:%d", ContainerDispatchPort))
									}

									return args
								})(),
								Env: common.CustomizeEnvvar(ctx, Component, common.MergeEnv(
									common.DefaultEnv(&ctx.Config),
									spicedbEnvVars(ctx),
								)),
								Ports: []corev1.ContainerPort{
									{
										ContainerPort: ContainerGRPCPort,
										Name:          ContainerGRPCName,
										Protocol:      *common.TCPProtocol,
									},
									{
										ContainerPort: ContainerHTTPPort,
										Name:          ContainerHTTPName,
										Protocol:      *common.TCPProtocol,
									},
									{
										ContainerPort: ContainerDashboardPort,
										Name:          ContainerDashboardName,
										Protocol:      *common.TCPProtocol,
									},
									{
										ContainerPort: ContainerDispatchPort,
										Name:          ContainerDispatchName,
										Protocol:      *common.TCPProtocol,
									},
									{
										ContainerPort: ContainerPrometheusPort,
										Name:          ContainterPrometheusName,
										Protocol:      *common.TCPProtocol,
									},
								},
								Resources: common.ResourceRequirements(ctx, Component, ContainerName, corev1.ResourceRequirements{
									Requests: corev1.ResourceList{
										"cpu":    resource.MustParse("1"),
										"memory": resource.MustParse("500M"),
									},
								}),
								SecurityContext: &corev1.SecurityContext{
									RunAsGroup:   pointer.Int64(65532),
									RunAsNonRoot: pointer.Bool(true),
									RunAsUser:    pointer.Int64(65532),
								},
								// Compare issue https://linear.app/gitpod/issue/EXP-906/spicedb-deployment-fails-in-gitpod-dedicated:
								//  - this should be a single grpc_health_probe-based readiness probe
								//  - but it started failing (with k8s 1.27.7 ?)
								//  - to unblock container startup, we split into readiness and liveness probes
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										// Exec: &v1.ExecAction{
										// 	Command: []string{"grpc_health_probe", "-v", fmt.Sprintf("-addr=localhost:%d", ContainerGRPCPort)},
										// },
										TCPSocket: &v1.TCPSocketAction{
											Port: intstr.FromInt(ContainerGRPCPort),
										},
									},
									InitialDelaySeconds: 1,
									// try again every 2 seconds
									PeriodSeconds: 2,
									// fail after 30 * 2 + 1 = 61
									FailureThreshold: 30,
									SuccessThreshold: 1,
									TimeoutSeconds:   1,
								},
								// Because we can't test readiness properly to not block startup, we use a liveness probe to test whether the cluster has come up
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										Exec: &v1.ExecAction{
											Command: []string{"grpc_health_probe", "-v", fmt.Sprintf("-addr=localhost:%d", ContainerGRPCPort)},
										},
									},
									InitialDelaySeconds: 10,
									PeriodSeconds:       10,
									FailureThreshold:    3,
									SuccessThreshold:    1,
									TimeoutSeconds:      1,
								},
								VolumeMounts: []v1.VolumeMount{
									bootstrapVolumeMount,
								},
							},
							*common.KubeRBACProxyContainer(ctx),
						},
						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
						Volumes: []v1.Volume{
							bootstrapVolume,
						},
					},
				},
			},
		},
	}, nil
}

func dbEnvVars(ctx *common.RenderContext) []corev1.EnvVar {
	return common.DatabaseEnv(&ctx.Config)
}

func dbWaiter(ctx *common.RenderContext) v1.Container {
	databaseWaiter := common.DatabaseMigrationWaiterContainer(ctx)
	// Use updated env-vars, which in the case cloud-sql-proxy override default db conf

	databaseWaiter.Env = dbEnvVars(ctx)

	return *databaseWaiter
}

func spicedbEnvVars(ctx *common.RenderContext) []corev1.EnvVar {
	cfg := getExperimentalSpiceDBConfig(ctx)
	if cfg == nil {
		return nil
	}

	return common.MergeEnv(
		dbEnvVars(ctx),
		[]corev1.EnvVar{
			{
				Name:  "SPICEDB_DATASTORE_CONN_URI",
				Value: "$(DB_USERNAME):$(DB_PASSWORD)@tcp($(DB_HOST):$(DB_PORT))/authorization?parseTime=true",
			},
			{
				Name: "SPICEDB_GRPC_PRESHARED_KEY",
				ValueFrom: &corev1.EnvVarSource{
					SecretKeyRef: &corev1.SecretKeySelector{
						LocalObjectReference: corev1.LocalObjectReference{
							Name: cfg.SecretRef,
						},
						Key: SecretPresharedKeyName,
					},
				},
			},
		},
	)
}