Kubernetes Multi-Runtime Cluster Playground

# Check runtime handlers available on each node
kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .status.nodeInfo.containerRuntimeVersion}{.}{"\n"}{end}{"\n"}{end}'

# More detailed view of runtime handlers per node
for node in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do
  echo "Node: $node"
  kubectl get node $node -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}{"\n"}'
  echo "---"
done

# Create RuntimeClass for runc
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: runc
handler: runc
EOF

# Deploy podinfo with runc across multiple nodes
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-runc
spec:
  replicas: 3
  selector:
    matchLabels:
      app: podinfo-runc
  template:
    metadata:
      labels:
        app: podinfo-runc
    spec:
      runtimeClassName: runc
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - podinfo-runc
              topologyKey: kubernetes.io/hostname
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
EOF

# Verify pods are scheduled across different nodes
kubectl get pods -l app=podinfo-runc -o wide

# Label nodes that support crun runtime
kubectl label node worker-containerd crun-support=enabled
kubectl label node worker-cri-o crun-support=enabled

# Create RuntimeClass for crun
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: crun
handler: crun
scheduling:
  nodeClassMap:
    crun: crun-support=enabled
EOF

# Deploy podinfo with crun using node affinity
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-crun
spec:
  replicas: 2
  selector:
    matchLabels:
      app: podinfo-crun
  template:
    metadata:
      labels:
        app: podinfo-crun
    spec:
      runtimeClassName: crun
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: crun-support
                operator: In
                values:
                - enabled
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - podinfo-crun
              topologyKey: kubernetes.io/hostname
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
EOF

# Verify pods are only scheduled on nodes with crun support
kubectl get pods -l app=podinfo-crun -o wide

# Taint the containerd node for runsc workloads
kubectl taint node worker-containerd runsc=enabled:NoSchedule

# Create RuntimeClass for runsc with toleration
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: runsc
handler: runsc
scheduling:
  tolerations:
  - key: runsc
    operator: Equal
    value: enabled
    effect: NoSchedule
  nodeClassMap:
    runsc: kubernetes.io/hostname=worker-containerd
EOF

# Deploy security-sensitive application with runsc
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-secure
spec:
  replicas: 2
  selector:
    matchLabels:
      app: podinfo-secure
  template:
    metadata:
      labels:
        app: podinfo-secure
    spec:
      runtimeClassName: runsc
      nodeSelector:
        kubernetes.io/hostname: worker-containerd
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
        securityContext:
          runAsNonRoot: true
          runAsUser: 1000
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
EOF

# All pods will be scheduled on the containerd node due to runsc requirement
kubectl get pods -l app=podinfo-secure -o wide

# Check cluster nodes and their container runtime
kubectl get nodes -o wide

# View cluster information
kubectl cluster-info

# Verify node labels for runtime scheduling
kubectl get nodes --show-labels | grep runtime

# Check which runtimes are available on each node
kubectl describe nodes | grep -A 5 -B 5 "Container Runtime"

# Create RuntimeClass for runc (default)
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: runc
handler: runc
EOF

# Create RuntimeClass for crun
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: crun
handler: crun
EOF

# Create RuntimeClass for runsc (gVisor)
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: runsc
handler: runsc
EOF

# List available RuntimeClasses
kubectl get runtimeclasses

# Describe a specific RuntimeClass
kubectl describe runtimeclass runc
kubectl describe runtimeclass crun
kubectl describe runtimeclass runsc

# Verify node capabilities before deploying workloads
kubectl get nodes -l runtime=containerd -o name  # Should show worker-containerd
kubectl get nodes -l runtime=cri-o -o name       # Should show worker-cri-o
kubectl get nodes -l runtime=docker -o name      # Should show worker-docker

# Create a simple deployment with default runtime
kubectl create deployment podinfo-default --image=ghcr.io/stefanprodan/podinfo --port=9898

# Check which node and runtime it's using
kubectl get pods -o wide
kubectl describe pod $(kubectl get pods -l app=podinfo-default -o jsonpath='{.items[0].metadata.name}')

# Create deployment with crun runtime (compatible with containerd and CRI-O)
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-crun
spec:
  replicas: 1
  selector:
    matchLabels:
      app: podinfo-crun
  template:
    metadata:
      labels:
        app: podinfo-crun
    spec:
      runtimeClassName: crun
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: runtime
                operator: In
                values: ["containerd", "cri-o"]  # crun works with containerd and CRI-O, but NOT Docker
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
EOF

# Verify the pod is using crun
kubectl get pods -l app=podinfo-crun -o wide
kubectl describe pod $(kubectl get pods -l app=podinfo-crun -o jsonpath='{.items[0].metadata.name}')

# Create deployment with runsc runtime for enhanced security (only compatible with containerd)
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-runsc
spec:
  replicas: 1
  selector:
    matchLabels:
      app: podinfo-runsc
  template:
    metadata:
      labels:
        app: podinfo-runsc
    spec:
      runtimeClassName: runsc
      nodeSelector:
        runtime: containerd  # runsc only works with containerd, NOT with CRI-O or Docker
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
EOF

# Verify the pod is using runsc
kubectl get pods -l app=podinfo-runsc -o wide
kubectl describe pod $(kubectl get pods -l app=podinfo-runsc -o jsonpath='{.items[0].metadata.name}')

# Create deployment targeting CRI-O node with crun
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-crio-crun
spec:
  replicas: 1
  selector:
    matchLabels:
      app: podinfo-crio-crun
  template:
    metadata:
      labels:
        app: podinfo-crio-crun
    spec:
      runtimeClassName: crun
      nodeSelector:
        runtime: cri-o  # Ensure it runs on CRI-O node
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
EOF

# Create deployment on Docker node (using cri-dockerd with runc only)
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: podinfo-docker
spec:
  replicas: 1
  selector:
    matchLabels:
      app: podinfo-docker
  template:
    metadata:
      labels:
        app: podinfo-docker
    spec:
      runtimeClassName: runc  # Docker only supports runc, not crun or runsc
      nodeSelector:
        runtime: docker  # Ensure it runs on cri-dockerd node
      containers:
      - name: podinfo
        image: ghcr.io/stefanprodan/podinfo
        ports:
        - containerPort: 9898
EOF

# Expose all deployments
kubectl expose deployment podinfo-default --port=9898 --name=podinfo-default-svc
kubectl expose deployment podinfo-crun --port=9898 --name=podinfo-crun-svc
kubectl expose deployment podinfo-runsc --port=9898 --name=podinfo-runsc-svc
kubectl expose deployment podinfo-crio-crun --port=9898 --name=podinfo-crio-crun-svc
kubectl expose deployment podinfo-docker --port=9898 --name=podinfo-docker-svc

# Wait for all pods to be ready
kubectl wait --for=condition=Ready pods --all --timeout=300s

# Test each application
kubectl get pods,services -o wide

# Test default runtime
curl http://$(kubectl get svc podinfo-default-svc -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')

# Test crun runtime
curl http://$(kubectl get svc podinfo-crun-svc -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')

# Test runsc runtime (gVisor)
curl http://$(kubectl get svc podinfo-runsc-svc -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')

# Test CRI-O with crun
curl http://$(kubectl get svc podinfo-crio-crun-svc -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')

# Test Docker runtime (via cri-dockerd)
curl http://$(kubectl get svc podinfo-docker-svc -o jsonpath='{.spec.clusterIP}:{.spec.ports[0].port}')

# Check runtime information on each worker node
kubectl get nodes -o custom-columns=NAME:.metadata.name,RUNTIME:.status.nodeInfo.containerRuntimeVersion

# Get detailed node information
kubectl describe node worker-containerd
kubectl describe node worker-cri-o
kubectl describe node worker-docker

# Switch to worker-containerd node terminal
# List containers with nerdctl
nerdctl --namespace k8s.io ps

# List containers with ctr
ctr --namespace k8s.io containers ls

# Check which runtime a container is using
ctr --namespace k8s.io info

# Switch to worker-cri-o node terminal
# List containers with crictl
crictl ps

# Get container runtime info
crictl info

# Switch to worker-docker node terminal
# List containers with docker
docker ps

# Get docker system info
docker system info

# Check cri-dockerd status
systemctl status cri-docker

# Check cri-dockerd configuration
cat /etc/systemd/system/cri-docker.service

# Create a test pod with runsc to verify gVisor isolation (only works on containerd)
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: gvisor-test
spec:
  runtimeClassName: runsc
  nodeSelector:
    runtime: containerd  # runsc only supported on containerd nodes
  containers:
  - name: test
    image: busybox
    command: ["sleep", "3600"]
EOF

# Exec into the pod and try to access host information
kubectl exec -it gvisor-test -- dmesg
kubectl exec -it gvisor-test -- cat /proc/version
kubectl exec -it gvisor-test -- ps aux

# Create identical workloads with different runtimes (all on containerd for fair comparison)
for runtime in runc crun runsc; do
  cat <<EOF | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
  name: perf-test-${runtime}
spec:
  template:
    metadata:
      labels:
        runtime-test: ${runtime}
    spec:
      runtimeClassName: ${runtime}
      nodeSelector:
        runtime: containerd  # Use containerd node for all tests since it supports all runtimes
      restartPolicy: Never
      containers:
      - name: perf-test
        image: busybox
        command: ["sh", "-c", "time dd if=/dev/zero of=/tmp/test bs=1M count=100; echo 'Runtime: ${runtime} completed'"]
EOF
done

# Monitor job completion times
kubectl get jobs -w
kubectl logs job/perf-test-runc
kubectl logs job/perf-test-crun
kubectl logs job/perf-test-runsc

# Example: Create a custom RuntimeClass for debugging
cat <<EOF | kubectl apply -f -
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: debug-runtime
handler: runc
overhead:
  podFixed:
    cpu: 10m
    memory: 10Mi
scheduling:
  nodeClassForScheduling: runtime-debug
  tolerations:
  - effect: NoSchedule
    key: runtime
    value: debug
EOF

# Get resource usage across different runtimes
kubectl top pods --all-namespaces

# Get detailed metrics for runtime comparison
kubectl get pods -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName,RUNTIME:.spec.runtimeClassName,STATUS:.status.phase

# Monitor pod distribution across runtimes
kubectl get pods --all-namespaces -o wide | grep -E "worker-(containerd|cri-o|docker)"

# Check pod events for scheduling errors
kubectl describe pod <pod-name>

# Look for events like:
# "0/4 nodes are available: 1 node(s) didn't match Pod's node affinity/selector"
# "RuntimeClass not supported by node"

# Check if RuntimeClass exists
kubectl get runtimeclasses

# Verify handler is supported on target nodes
kubectl describe node <node-name> | grep "Container Runtime"

# Check container logs for runtime-specific errors
kubectl logs <pod-name>

# Check node logs for runtime issues
kubectl describe node <node-name>

# On the node itself, check runtime status:
# For containerd nodes:
systemctl status containerd

# For CRI-O nodes:
systemctl status crio

# For Docker nodes:
systemctl status docker
systemctl status cri-docker

# Test runtime compatibility before deploying
kubectl run test-runc --image=busybox --rm -it --restart=Never \
  --overrides='{"spec":{"runtimeClassName":"runc","nodeSelector":{"runtime":"containerd"}}}' \
  -- echo "runc on containerd works"

kubectl run test-crun --image=busybox --rm -it --restart=Never \
  --overrides='{"spec":{"runtimeClassName":"crun","nodeSelector":{"runtime":"containerd"}}}' \
  -- echo "crun on containerd works"

kubectl run test-runsc --image=busybox --rm -it --restart=Never \
  --overrides='{"spec":{"runtimeClassName":"runsc","nodeSelector":{"runtime":"containerd"}}}' \
  -- echo "runsc on containerd works"

# Clean up test deployments
kubectl delete deployment podinfo-default podinfo-crun podinfo-runsc podinfo-crio-crun podinfo-docker

# Clean up services
kubectl delete service podinfo-default-svc podinfo-crun-svc podinfo-runsc-svc podinfo-crio-crun-svc podinfo-docker-svc

# Clean up test pods and jobs
kubectl delete pod gvisor-test
kubectl delete job perf-test-runc perf-test-crun perf-test-runsc