Step 1: Install k3s + register the NVIDIA runtime
Install k3s in server mode. We write out the exact command you'd run on the node, plus the containerd config snippet that tells k3s how to launch pods under the NVIDIA runtime.
Why the config path matters: k3s uses a distinct containerd under /var/lib/rancher/k3s/agent/etc/containerd/. The default NVIDIA installer patches /etc/containerd/config.toml which k3s ignores. You must either configure k3s to use the system containerd, or drop a template file into k3s's path. We do the latter — cleaner, fully managed by k3s.
from pathlib import Path
from IPython.display import Code, display
WORKSPACE = Path('/home/labuser/workspace')
# Load the k3s install script + containerd config template from the workspace.
# Authoring them as real files (instead of Python triple-quoted strings) lets you
# `shellcheck scripts/install_k3s.sh`, diff them in git, and re-use them outside
# the notebook.
k3s_install_script = (WORKSPACE / 'scripts/install_k3s.sh').read_text()
nvidia_runtime_config = (WORKSPACE / 'containerd/config.toml.tmpl').read_text()
display(Code(filename=str(WORKSPACE / 'scripts/install_k3s.sh'), language='bash'))
display(Code(filename=str(WORKSPACE / 'containerd/config.toml.tmpl'), language='toml'))
# Post-install sanity checks — each one catches a real failure mode
post_install_checks = [
{
'step': 'node ready',
'command': 'kubectl get nodes -o wide',
'expected_output': 'STATUS column shows Ready; Container-Runtime shows containerd://... on the node',
},
{
'step': 'containerd knows nvidia',
'command': 'sudo grep -A3 "runtimes.nvidia" /var/lib/rancher/k3s/agent/etc/containerd/config.toml',
'expected_output': 'block present with BinaryName = "/usr/bin/nvidia-container-runtime"',
},
{
'step': 'host driver visible',
'command': 'nvidia-smi',
'expected_output': 'lists the GPU, driver version, CUDA version, no processes at idle',
},
{
'step': 'test pod sees GPU',
'command': 'kubectl run gpu-smoke --rm -it --image=nvidia/cuda:12.4.1-base-ubuntu22.04 '
'--overrides=\'{"spec":{"runtimeClassName":"nvidia","containers":[{"name":"gpu-smoke","image":"nvidia/cuda:12.4.1-base-ubuntu22.04","command":["nvidia-smi"],"resources":{"limits":{"nvidia.com/gpu":1}}}]}}\' -- nvidia-smi',
'expected_output': 'pod runs to completion and prints nvidia-smi output listing the GPU',
},
{
'step': 'kubeconfig permissions',
'command': 'ls -la /etc/rancher/k3s/k3s.yaml',
'expected_output': 'mode 644 (or 600 if you ran without --write-kubeconfig-mode) — NOT world-writable',
},
]
print(f"\n{len(post_install_checks)} post-install checks:")
for chk in post_install_checks:
print(f" [{chk['step']}] {chk['command'][:80]}...")
print(f" expect: {chk['expected_output'][:80]}")
