timeout_controller_test.go

// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License-AGPL.txt in the project root for license information.

package controllers

import (
	"time"

	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
	workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1"
	"github.com/google/uuid"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/pointer"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	// . "github.com/onsi/ginkgo/extensions/table"
)

var _ = Describe("TimeoutController", func() {
	Context("timeouts", func() {
		var (
			now        = time.Now()
			conf       = newTestConfig()
			r          *TimeoutReconciler
			fakeClient client.Client
		)
		BeforeEach(func() {
			var err error
			// Use a fake client instead of the envtest's k8s client, such that we can add objects
			// with custom CreationTimestamps and check timeout logic.
			fakeClient = fake.NewClientBuilder().WithStatusSubresource(&workspacev1.Workspace{}).WithScheme(k8sClient.Scheme()).Build()
			r, err = NewTimeoutReconciler(fakeClient, record.NewFakeRecorder(100), conf, &fakeMaintenance{enabled: false})
			Expect(err).ToNot(HaveOccurred())
		})

		type testCase struct {
			phase             workspacev1.WorkspacePhase
			lastActivityAgo   *time.Duration
			age               time.Duration
			customTimeout     *time.Duration
			customMaxLifetime *time.Duration
			update            func(ws *workspacev1.Workspace)
			updateStatus      func(ws *workspacev1.Workspace)
			expectTimeout     bool
		}
		DescribeTable("workspace timeouts",
			func(tc testCase) {
				By("creating a workspace")
				ws := newWorkspace(uuid.NewString(), "default")
				ws.CreationTimestamp = metav1.NewTime(now.Add(-tc.age))

				if tc.lastActivityAgo != nil {
					now := metav1.NewTime(now.Add(-*tc.lastActivityAgo))
					ws.Status.LastActivity = &now
				}

				Expect(fakeClient.Create(ctx, ws)).To(Succeed())

				updateObjWithRetries(fakeClient, ws, false, func(ws *workspacev1.Workspace) {
					if tc.customTimeout != nil {
						ws.Spec.Timeout.Time = &metav1.Duration{Duration: *tc.customTimeout}
					}
					if tc.customMaxLifetime != nil {
						ws.Spec.Timeout.MaximumLifetime = &metav1.Duration{Duration: *tc.customMaxLifetime}
					}
					if tc.update != nil {
						tc.update(ws)
					}
				})
				updateObjWithRetries(fakeClient, ws, true, func(ws *workspacev1.Workspace) {
					ws.Status.Phase = tc.phase
					if tc.updateStatus != nil {
						tc.updateStatus(ws)
					}
				})

				// Run the timeout controller for this workspace.
				By("running the TimeoutController reconcile()")
				_, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: ws.Name, Namespace: ws.Namespace}})
				Expect(err).ToNot(HaveOccurred())

				if tc.expectTimeout {
					expectTimeout(fakeClient, ws)
				} else {
					expectNoTimeout(fakeClient, ws)
				}
			},
			Entry("should timeout creating workspace", testCase{
				phase:         workspacev1.WorkspacePhaseCreating,
				age:           10 * time.Hour,
				expectTimeout: true,
			}),
			Entry("shouldn't timeout active workspace", testCase{
				phase:           workspacev1.WorkspacePhaseRunning,
				lastActivityAgo: pointer.Duration(1 * time.Minute),
				age:             10 * time.Hour,
				expectTimeout:   false,
			}),
			Entry("should timeout inactive workspace", testCase{
				phase:           workspacev1.WorkspacePhaseRunning,
				lastActivityAgo: pointer.Duration(2 * time.Hour),
				age:             10 * time.Hour,
				expectTimeout:   true,
			}),
			Entry("should timeout inactive workspace with custom timeout", testCase{
				phase: workspacev1.WorkspacePhaseRunning,
				// Use a lastActivity that would not trigger the default timeout, but does trigger the custom timeout.
				lastActivityAgo: pointer.Duration(time.Duration(conf.Timeouts.RegularWorkspace / 2)),
				customTimeout:   pointer.Duration(time.Duration(conf.Timeouts.RegularWorkspace / 3)),
				age:             10 * time.Hour,
				expectTimeout:   true,
			}),
			Entry("should timeout closed workspace", testCase{
				phase: workspacev1.WorkspacePhaseRunning,
				updateStatus: func(ws *workspacev1.Workspace) {
					ws.Status.Conditions = wsk8s.AddUniqueCondition(ws.Status.Conditions, metav1.Condition{
						Type:               string(workspacev1.WorkspaceConditionClosed),
						LastTransitionTime: metav1.Now(),
						Status:             metav1.ConditionTrue,
					})
				},
				age:             5 * time.Hour,
				lastActivityAgo: pointer.Duration(10 * time.Minute),
				expectTimeout:   true,
			}),
			Entry("should timeout headless workspace", testCase{
				phase: workspacev1.WorkspacePhaseRunning,
				update: func(ws *workspacev1.Workspace) {
					ws.Spec.Type = workspacev1.WorkspaceTypePrebuild
				},
				age:             2 * time.Hour,
				lastActivityAgo: nil,
				expectTimeout:   true,
			}),
			Entry("should timeout workspace with no custom lifetime", testCase{
				phase:           workspacev1.WorkspacePhaseRunning,
				age:             50 * time.Hour,
				lastActivityAgo: pointer.Duration(1 * time.Minute),
				expectTimeout:   true,
			}),
			Entry("should timeout workspace with custom lifetime", testCase{
				phase:             workspacev1.WorkspacePhaseRunning,
				age:               12 * time.Hour,
				customMaxLifetime: pointer.Duration(8 * time.Hour),
				lastActivityAgo:   pointer.Duration(1 * time.Minute),
				expectTimeout:     true,
			}),
			Entry("should timeout after controller restart if no FirstUserActivity", testCase{
				phase:           workspacev1.WorkspacePhaseRunning,
				age:             5 * time.Hour,
				lastActivityAgo: nil, // No last activity recorded yet after controller restart.
				expectTimeout:   true,
			}),
			Entry("should timeout eventually with no user activity after controller restart", testCase{
				phase: workspacev1.WorkspacePhaseRunning,
				updateStatus: func(ws *workspacev1.Workspace) {
					ws.Status.Conditions = wsk8s.AddUniqueCondition(ws.Status.Conditions, metav1.Condition{
						Type:               string(workspacev1.WorkspaceConditionFirstUserActivity),
						Status:             metav1.ConditionTrue,
						LastTransitionTime: metav1.NewTime(now.Add(-5 * time.Hour)),
					})
				},
				age:             5 * time.Hour,
				lastActivityAgo: nil,
				expectTimeout:   true,
			}),
		)
	})

	Context("reconciliation", func() {
		var r *TimeoutReconciler
		BeforeEach(func() {
			var err error
			r, err = NewTimeoutReconciler(k8sClient, record.NewFakeRecorder(100), newTestConfig(), &fakeMaintenance{enabled: false})
			Expect(err).ToNot(HaveOccurred())
		})

		It("should requeue timeout reconciles", func() {
			ws := newWorkspace(uuid.NewString(), "default")
			_ = createWorkspaceExpectPod(ws)

			res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: ws.Name, Namespace: ws.Namespace}})
			Expect(err).To(BeNil())
			Expect(r.reconcileInterval).ToNot(BeZero(), "reconcile interval should be > 0, otherwise events will not requeue")
			Expect(res.RequeueAfter).To(Equal(r.reconcileInterval))
		})

		It("should not requeue when resource is not found", func() {
			res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "does-not-exist", Namespace: "default"}})
			Expect(err).ToNot(HaveOccurred(), "not-found errors should not be returned")
			Expect(res.Requeue).To(BeFalse())
			Expect(res.RequeueAfter).To(BeZero())
		})

		It("should return an error other than not-found", func() {
			// Create a different error than "not-found", easiest is to provide an empty name which returns an "invalid request".
			_, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: "", Namespace: "default"}})
			Expect(err).To(HaveOccurred(), "should return error and requeue")
		})
	})
})

func expectNoTimeout(c client.Client, ws *workspacev1.Workspace) {
	GinkgoHelper()
	By("expecting controller to not timeout workspace")
	Consistently(func(g Gomega) {
		g.Expect(c.Get(ctx, types.NamespacedName{Name: ws.Name, Namespace: ws.Namespace}, ws)).To(Succeed())
		g.Expect(wsk8s.GetCondition(ws.Status.Conditions, string(workspacev1.WorkspaceConditionTimeout))).To(BeNil())
	}, duration, interval).Should(Succeed())
}

func expectTimeout(c client.Client, ws *workspacev1.Workspace) {
	GinkgoHelper()
	By("expecting controller to timeout workspace")
	Eventually(func(g Gomega) {
		g.Expect(c.Get(ctx, types.NamespacedName{Name: ws.Name, Namespace: ws.Namespace}, ws)).To(Succeed())
		cond := wsk8s.GetCondition(ws.Status.Conditions, string(workspacev1.WorkspaceConditionTimeout))
		g.Expect(cond).ToNot(BeNil())
		g.Expect(cond.Status).To(Equal(metav1.ConditionTrue))
	}, timeout, interval).Should(Succeed())
}