k8s.yaml

# Deploy kubernetes via kubeadm.
# $ limactl start ./k8s.yaml
# $ limactl shell k8s kubectl

# It can be accessed from the host by exporting the kubeconfig file;
# the ports are already forwarded automatically by lima:
#
# $ export KUBECONFIG=$(limactl list k8s --format 'unix://{{.Dir}}/copied-from-guest/kubeconfig.yaml')
# $ kubectl get no
# NAME       STATUS   ROLES                  AGE   VERSION
# lima-k8s   Ready    control-plane,master   44s   v1.22.3

# A multi-node cluster can be created by starting multiple instances of this template
# connected via the `lima:user-v2` network.
#
# $ limactl start --name k8s-0 --network lima:user-v2 template:k8s
# $ limactl shell k8s-0 sudo kubeadm token create --print-join-command
# (The parameters for the start command printed here)
#
# $ limactl start --name k8s-1 --network lima:user-v2 template:k8s \
#                 --set '.param.url="https://<ADDRESS_FROM_ABOVE>" | .param.token="<TOKEN_FROM_ABOVE>" | \
#                        .param.discoveryTokenCaCertHash="<DISCOVERY_TOKEN_CA_CERT_HASH_FROM_ABOVE>"'

minimumLimaVersion: 2.0.0

base: template:_images/ubuntu-lts

# Mounts are disabled in this template, but can be enabled optionally.
mounts: []
containerd:
  system: true
  user: false
provision:
# See <https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/>
- mode: system
  script: |
    #!/bin/bash
    set -eux -o pipefail
    command -v kubeadm >/dev/null 2>&1 && exit 0
    # Install and configure prerequisites
    cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
    overlay
    br_netfilter
    EOF
    modprobe overlay
    modprobe br_netfilter
    cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
    net.bridge.bridge-nf-call-iptables  = 1
    net.ipv4.ip_forward                 = 1
    net.bridge.bridge-nf-call-ip6tables = 1
    EOF
    sysctl --system
    # Installing kubeadm, kubelet and kubectl
    export DEBIAN_FRONTEND=noninteractive
    apt-get update
    apt-get install -y apt-transport-https ca-certificates curl
    VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt | sed -e 's/v//' | cut -d'.' -f1-2)
    echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
    curl -fsSL https://pkgs.k8s.io/core:/stable:/v${VERSION}/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
    apt-get update
    apt-get install -y kubelet kubeadm kubectl && apt-mark hold kubelet kubeadm kubectl
    systemctl enable --now kubelet
# See <https://kubernetes.io/docs/setup/production-environment/container-runtimes/>
- mode: system
  script: |
    #!/bin/bash
    set -eux -o pipefail
    [ -e /etc/containerd/conf.d/k8s.toml ] && exit 0
    mkdir -p /etc/containerd/conf.d
    # Configuring the systemd cgroup driver
    # Overriding the sandbox (pause) image
    cat <<EOF >/etc/containerd/conf.d/k8s.toml
    version = 2
    [plugins]
      [plugins."io.containerd.grpc.v1.cri"]
        sandbox_image = "$(kubeadm config images list | grep pause | sort -r | head -n1)"
        [plugins."io.containerd.grpc.v1.cri".containerd]
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
              runtime_type = "io.containerd.runc.v2"
              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
                SystemdCgroup = true
      [plugins."io.containerd.cri.v1.runtime".cni]
        bin_dirs = ["/usr/local/libexec/cni","/opt/cni/bin"]
    EOF
    systemctl restart containerd
# See <https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/>
- mode: system
  script: |
    #!/bin/bash
    set -eux -o pipefail
    test -e /etc/kubernetes/admin.conf && exit 0
    export KUBECONFIG=/etc/kubernetes/admin.conf
    {{if not ( and .Param.url .Param.token )}}
    systemctl stop kubelet
    kubeadm config images list
    kubeadm config images pull --cri-socket=unix:///run/containerd/containerd.sock
    systemctl start kubelet
    # Initializing your control-plane node
    cat <<EOF >kubeadm-config.yaml
    kind: InitConfiguration
    apiVersion: kubeadm.k8s.io/v1beta4
    nodeRegistration:
      criSocket: unix:///run/containerd/containerd.sock
    ---
    kind: ClusterConfiguration
    apiVersion: kubeadm.k8s.io/v1beta4
    apiServer:
      certSANs: # --apiserver-cert-extra-sans
      - "127.0.0.1"
    networking:
      podSubnet: "10.244.0.0/16" # --pod-network-cidr
    ---
    kind: KubeletConfiguration
    apiVersion: kubelet.config.k8s.io/v1beta1
    cgroupDriver: systemd
    EOF
    kubeadm init --config kubeadm-config.yaml
    {{else}}
    cat <<EOF >kubeadm-config.yaml
    kind: JoinConfiguration
    apiVersion: kubeadm.k8s.io/v1beta4
    nodeRegistration:
      criSocket: unix:///run/containerd/containerd.sock
    EOF
    kubeadm join --config kubeadm-config.yaml {{.Param.url}} --token {{.Param.token}} \
        --discovery-token-ca-cert-hash {{.Param.discoveryTokenCaCertHash}}
    {{end}}

    {{if not ( and .Param.url .Param.token )}}
    # Installing a Pod network add-on
    kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.27.4/kube-flannel.yml
    # Control plane node isolation
    kubectl taint nodes --all node-role.kubernetes.io/control-plane-
    # Symlink the kubeconfig file to the default location for kubectl
    mkdir -p /root/.kube && ln -sf $KUBECONFIG /root/.kube/config
    # Replace the server address with localhost, so that it works from the host.
    # The original kubeconfig is kept unmodified, so that `kubeadm token create --print-join-command`
    # can still print the reachable address.
    sed -e "/server:/ s|https://.*:\([0-9]*\)$|https://127.0.0.1:\1|" $KUBECONFIG >/root/.kube/config.localhost
    {{end}}
- mode: system
  script: |
    #!/bin/bash
    set -eux -o pipefail
    export KUBECONFIG=/etc/kubernetes/admin.conf
    mkdir -p {{.Home}}/.kube
    cp -f $KUBECONFIG {{.Home}}/.kube/config
    chown -R {{.User}} {{.Home}}/.kube
probes:
- description: "kubeadm to be installed"
  script: |
    #!/bin/bash
    set -eux -o pipefail
    if ! timeout 30s bash -c "until command -v kubeadm >/dev/null 2>&1; do sleep 3; done"; then
      echo >&2 "kubeadm is not installed yet"
      exit 1
    fi
  hint: |
    See "/var/log/cloud-init-output.log" in the guest
- description: "kubernetes images to be pulled"
  script: |
    #!/bin/bash
    set -eux -o pipefail
    {{if not ( and .Param.url .Param.token )}}
    if ! timeout 30s bash -c "images=\"$(kubeadm config images list)\"; until for image in \$images; do sudo ctr -n k8s.io image inspect \$image >/dev/null; done; do sleep 3; done"; then
      echo >&2 "k8s images are not pulled yet"
      exit 1
    fi
    {{end}}
- description: "kubeadm to be completed"
  script: |
    #!/bin/bash
    set -eux -o pipefail
    {{if not ( and .Param.url .Param.token )}}
    if ! timeout 300s bash -c "until test -f /etc/kubernetes/admin.conf; do sleep 3; done"; then
      echo >&2 "k8s is not running yet"
      exit 1
    fi
    {{else}}
    # create an empty file so that the "copyToHost" does not fail
    sudo mkdir -p /root/.kube && sudo touch /root/.kube/config.localhost
    {{end}}
  hint: |
    The k8s kubeconfig file has not yet been created.
- description: "kubernetes cluster to be running"
  script: |
    #!/bin/bash
    set -eux -o pipefail
    {{if not ( and .Param.url .Param.token )}}
    if ! timeout 300s bash -c "until kubectl version >/dev/null 2>&1; do sleep 3; done"; then
      echo >&2 "kubernetes cluster is not up and running yet"
      exit 1
    fi
    {{end}}
- description: "coredns deployment to be running"
  script: |
    #!/bin/bash
    set -eux -o pipefail
    {{if not ( and .Param.url .Param.token )}}
    kubectl wait -n kube-system --timeout=180s --for=condition=available deploy coredns
    {{end}}
copyToHost:
- guest: "/root/.kube/config.localhost"
  host: "{{.Dir}}/copied-from-guest/kubeconfig.yaml"
  deleteOnStop: true
message: |
  {{- if not ( and .Param.url .Param.token )}}
  To run `kubectl` on the host (assumes kubectl is installed), run the following commands:
  ------
  export KUBECONFIG="{{.Dir}}/copied-from-guest/kubeconfig.yaml"
  kubectl ...
  ------
  {{end -}}
param:
  url: ""
  token: ""
  discoveryTokenCaCertHash: ""