feat(nvsnap): shared-token authentication for the agent API

src/compute-plane-services/nvsnap/cmd/agent/main.go

@@ -71,6 +71,8 @@ func main() {
 		"NvSnap-server base URL for peer-fanout catalog lookups (e.g. http://nvsnap-server.nvsnap-system.svc.cluster.local:8080). Empty disables cross-node cascade.")
 	flag.StringVar(&config.NodeIP, "node-ip", os.Getenv("HOST_IP"),
 		"This agent's reachable address from peers (downward API status.hostIP when hostNetwork:true). Empty disables peer registration.")
+	flag.StringVar(&config.AdvertiseIP, "advertise-ip", os.Getenv("POD_IP"),
+		"Address peers dial to reach this agent (downward API status.podIP; equals the node IP under hostNetwork). Falls back to --node-ip when empty. See GH #490.")
 	flag.StringVar(&config.BlobStoreURL, "blob-store-url", os.Getenv("NVSNAP_BLOB_STORE_URL"),
 		"NvSnap-blobstore base URL for Phase 5d.2 durable backstop (e.g. http://nvsnap-blobstore.nvsnap-system.svc.cluster.local:9000). Empty disables capture-side upload AND cascade tier-3 fallback.")
 	// Cross-cluster replication (docs/design/cross-cluster-replication.md).
@@ -90,6 +92,11 @@ func main() {
 	flag.StringVar(&config.FSStorePath, "fsstore-path", os.Getenv("NVSNAP_FSSTORE_PATH"),
 		"Path to a shared filesystem mounted on every node (Lustre/Weka/EFS/Filestore/NFS). When set, captures are published here and the restore cascade copies from this path before peer fanout. Empty disables.")
 	flag.StringVar(&config.ListenAddr, "listen", ":8081", "Listen address")
+	// The token itself is env-only, never a flag: flag values show up in the
+	// pod spec and in `ps`, and this is a credential.
+	var authMode string
+	flag.StringVar(&authMode, "auth-mode", os.Getenv("NVSNAP_AGENT_AUTH_MODE"),
+		"Agent API authentication: disabled (default), permissive (check, log failures, still serve), or required (401). Token comes from NVSNAP_AGENT_TOKEN. See GH #486.")
 	flag.StringVar(&config.CheckpointDir, "checkpoint-dir", "/var/lib/nvsnap/checkpoints", "Checkpoint storage directory (in-agent-container path)")
 	flag.StringVar(&config.CheckpointHostDir, "checkpoint-host-dir", "/var/lib/containerd/nvsnap-checkpoints", "Host path that backs --checkpoint-dir (must match the DaemonSet hostPath mount; used to translate paths for the capture-write writer Job)")
 	flag.StringVar(&config.CRIUPath, "criu-path", "/usr/local/sbin/criu", "Path to CRIU binary (on host filesystem)")
@@ -183,6 +190,8 @@ func main() {
 		"Strategy for restore-side overlay mount prep: inline (do mounts during admission, default) or init-container (delegate to nvsnap-mount-prep init container on the restored pod)")
 	flag.StringVar(&config.Webhook.MountPrepInitImage, "webhook-mount-prep-init-image", "",
 		"Image ref for the nvsnap-mount-prep init container injected when --webhook-restore-prep-strategy=init-container. Must contain /nvsnap-mount-prep (the agent image satisfies this).")
+	flag.StringVar(&config.Webhook.AgentBaseURL, "webhook-agent-base-url", os.Getenv("NVSNAP_WEBHOOK_AGENT_BASE_URL"),
+		"Base URL the injected nvsnap-mount-prep init container uses to reach its node-local agent. Empty uses http://$(NVSNAP_HOST_IP):<port>, which requires hostPort. Set to the internalTrafficPolicy:Local Service under pod networking. See GH #490.")
 	flag.IntVar(&config.Webhook.AgentHostPort, "webhook-agent-host-port", 8081,
 		"Port the nvsnap-mount-prep init container reaches the agent on (matches --listen and the agent DaemonSet's hostPort).")
 
@@ -204,6 +213,19 @@ func main() {
 		"imagePullSecret name for the mount-holder pod (created by operators in the workload namespace). Defaults to nvsnap-agent-pull; set to '-' to disable.")
 
 	flag.Parse()
+
+	// Fail startup on a bad mode rather than falling back to disabled: an
+	// operator who typo'd --auth-mode should hear about it now, not discover
+	// months later that the API was open the whole time.
+	var authErr error
+	if config.AuthMode, authErr = agent.ParseAuthMode(authMode); authErr != nil {
+		logrus.WithError(authErr).Fatal("invalid --auth-mode")
+	}
+	config.AuthToken = os.Getenv("NVSNAP_AGENT_TOKEN")
+	if config.AuthMode != agent.AuthDisabled && config.AuthToken == "" {
+		logrus.Fatalf("--auth-mode=%s requires NVSNAP_AGENT_TOKEN to be set", config.AuthMode)
+	}
+
 	config.RootfsCapture.WarmupDelay = time.Duration(rootfsWarmupSec) * time.Second
 	for _, b := range strings.Split(replicationPeerBuckets, ",") {
 		if b = strings.TrimSpace(b); b != "" {

src/compute-plane-services/nvsnap/cmd/nvsnap-mount-prep/main.go

@@ -40,6 +40,9 @@ limitations under the License.
 //	NVSNAP_POD_UID         (required) downward API: metadata.uid
 //	NVSNAP_RESTORE_HASH    (required) full sha256 of the capture
 //	NVSNAP_AGENT_URL       (required) e.g. http://$(HOST_IP):8081
+//	NVSNAP_AGENT_TOKEN     (optional) bearer token for the agent API (GH #486);
+//	                       empty sends no header, which is correct while the
+//	                       agent still runs with auth disabled
 //	NVSNAP_CAPTURE_NODE    (optional) where capture data lives; empty=this node
 //	NVSNAP_PREP_MOUNTS     (required) JSON-encoded []VolumeMeta from the manifest
 //	NVSNAP_PREP_DEADLINE   (optional) duration; default 15m
@@ -196,6 +199,7 @@ func startWithRetry(agentURL string, req prepRequest) error {
 			return err
 		}
 		httpReq.Header.Set("Content-Type", "application/json")
+		setAgentAuth(httpReq)
 		resp, err := http.DefaultClient.Do(httpReq)
 		if err != nil {
 			lastErr = err
@@ -220,6 +224,7 @@ func startWithRetry(agentURL string, req prepRequest) error {
 
 func getStatus(agentURL, podUID string) (*prepStatus, error) {
 	httpReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, agentURL+"/v1/restore/prep/"+podUID, http.NoBody)
+	setAgentAuth(httpReq)
 	if err != nil {
 		return nil, err
 	}
@@ -244,3 +249,15 @@ func getStatus(agentURL, podUID string) (*prepStatus, error) {
 	}
 	return &s, nil
 }
+
+// setAgentAuth attaches the agent API bearer token when one is configured.
+// Empty is the normal state until the operator turns auth on, and sending no
+// header is exactly what a disabled or permissive agent expects. See GH #486.
+func setAgentAuth(r *http.Request) {
+	if r == nil {
+		return
+	}
+	if tok := os.Getenv("NVSNAP_AGENT_TOKEN"); tok != "" {
+		r.Header.Set("Authorization", "Bearer "+tok)
+	}
+}

src/compute-plane-services/nvsnap/cmd/nvsnap-server/main.go

@@ -106,6 +106,11 @@ func run(cmd *cobra.Command, args []string) error {
 		Address:      address,
 		AgentPort:    agentPort,
 		BlobstoreURL: blobstoreURL,
+		// Env rather than a flag: the chart projects it straight from the
+		// nvsnap-agent-token Secret, and a flag would put the credential in
+		// the process argv where any pod-reading client can see it.
+		// Empty when auth is off, which sends no header (nvsnap#736).
+		AgentToken: os.Getenv("NVSNAP_AGENT_TOKEN"),
 	}, kubeClient, dynClient, catalog)
 
 	// Embed React UI — serves at / with SPA fallback

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-daemonset.yaml

@@ -42,9 +42,9 @@ spec:
         {{- toYaml .Values.agent.tolerations | nindent 8 }}
       hostPID: {{ .Values.agent.hostPID }}
       hostNetwork: {{ .Values.agent.hostNetwork }}
-      # ClusterFirstWithHostNet so Service DNS still resolves under
-      # hostNetwork (nvsnap-server.<ns>.svc, nvsnap-blobstore.<ns>.svc).
-      dnsPolicy: ClusterFirstWithHostNet
+      # ClusterFirstWithHostNet is only correct under hostNetwork; with pod
+      # networking it is wrong (it points resolution at the node's resolv.conf).
+      dnsPolicy: {{ if .Values.agent.hostNetwork }}ClusterFirstWithHostNet{{ else }}ClusterFirst{{ end }}
       serviceAccountName: nvsnap-agent
       {{- include "nvsnap.imagePullSecrets" . | nindent 6 }}
       initContainers:
@@ -82,6 +82,18 @@ spec:
           image: {{ include "nvsnap.agent.image" . }}
           imagePullPolicy: {{ .Values.agent.image.pullPolicy }}
           args:
+            {{- if .Values.agent.auth.enabled }}
+            # permissive counts and logs unauthenticated callers but still
+            # serves them; required returns 401. Roll out on permissive until
+            # nvsnap_agent_auth_total{result="missing"} is zero.
+            - --auth-mode={{ .Values.agent.auth.mode }}
+            {{- end }}
+            {{- if not .Values.agent.hostNetwork }}
+            # Pod networking: the init container reaches its node-local agent
+            # through the internalTrafficPolicy:Local Service instead of the
+            # node IP, so no hostPort is needed (GH #490).
+            - --webhook-agent-base-url=http://nvsnap-agent-local.{{ .Release.Namespace }}.svc.cluster.local:8081
+            {{- end }}
             - --cuda-checkpoint-path=/criu-bundle/cuda-checkpoint
             - --criu-path=/criu-bundle/criu
             # Translate in-container --checkpoint-dir to the host path
@@ -168,6 +180,24 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.hostIP
+            # POD_IP is what peers dial (--advertise-ip). Under hostNetwork
+            # kubelet reports status.podIP as the node IP, so this is correct
+            # in both network modes and needs no conditional (GH #490).
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            {{- if .Values.agent.auth.enabled }}
+            # Shared bearer token for the agent API (GH #486). Env rather than
+            # a flag: flag values are visible in the pod spec and in `ps`.
+            # The agent uses it both to verify inbound requests and to sign
+            # its own peer calls.
+            - name: NVSNAP_AGENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: nvsnap-agent-token
+                  key: token
+            {{- end }}
             {{- if .Values.server.enabled }}
             - name: NVSNAP_CATALOG_URL
               value: "http://nvsnap-server.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.server.service.port }}"
@@ -225,7 +255,13 @@ spec:
             {{- end }}
           ports:
             - containerPort: 8081
+              {{- if .Values.agent.hostNetwork }}
+              # Binds the API to every node's IP. Only declared under
+              # hostNetwork; with pod networking peers dial the pod IP and
+              # same-node callers use the internalTrafficPolicy:Local
+              # Service, so no node-wide listener is needed (GH #490).
               hostPort: 8081
+              {{- end }}
               name: http-api
             {{- if .Values.webhook.enabled }}
             - containerPort: 8443
@@ -386,3 +422,33 @@ spec:
       name: http-api
   clusterIP: None
 {{- end }}
+
+{{- if not .Values.agent.hostNetwork }}
+---
+# Node-local Service: the pod-network replacement for hostPort (GH #490).
+#
+# Callers that must reach the agent on THEIR OWN node -- the nvsnap-mount-prep
+# init container is the one that matters -- used to do it via
+# http://$(status.hostIP):8081, which requires the API to be bound to every
+# node's IP. internalTrafficPolicy:Local is the Kubernetes-native way to say
+# the same thing: this ClusterIP only ever routes to the endpoint on the
+# calling node, and has no node-IP listener at all.
+#
+# The headless nvsnap-agent Service above stays for tools that want to address
+# a specific agent; this one is for "whichever agent is on my node".
+apiVersion: v1
+kind: Service
+metadata:
+  name: nvsnap-agent-local
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nvsnap.agent.labels" . | nindent 4 }}
+spec:
+  selector:
+    {{- include "nvsnap.agent.selectorLabels" . | nindent 4 }}
+  internalTrafficPolicy: Local
+  ports:
+    - port: 8081
+      targetPort: 8081
+      name: http-api
+{{- end }}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/agent-token-secret.yaml

@@ -0,0 +1,88 @@
+{{- if .Values.agent.auth.enabled }}
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared bearer token for the agent HTTP API (GH #486).
+#
+# The agent API is the control surface of a privileged process and the
+# DaemonSet binds it to every node's IP, so it needs authentication in the
+# request path. This Secret holds the token both the agent (to verify) and its
+# callers (to present) read.
+#
+# Generated once and preserved across upgrades: `helm upgrade` re-renders every
+# template, so a freshly random token on each upgrade would rotate the
+# credential out from under running callers and cause a self-inflicted outage
+# mid-rollout. The lookup below reuses the existing value when the Secret is
+# already present. Set agent.auth.token explicitly to manage it yourself (or to
+# rotate deliberately).
+{{- $ns := .Release.Namespace }}
+{{- $name := "nvsnap-agent-token" }}
+{{- $existing := lookup "v1" "Secret" $ns $name }}
+{{- $token := "" }}
+{{- if .Values.agent.auth.token }}
+{{-   $token = .Values.agent.auth.token | b64enc }}
+{{- else if and $existing $existing.data $existing.data.token }}
+{{-   $token = $existing.data.token }}
+{{- else }}
+# Reached on a first install, and on any client-side render -- `lookup` returns
+# empty without cluster access, so an existing Secret is invisible here and
+# indistinguishable from a missing one.
+#
+# That makes `helm template` non-deterministic: every run mints a different
+# token, and `helm template | kubectl apply` would rotate the credential out
+# from under running agents, which keep the old value in their environment
+# while fresh mount-prep pods get the new one and 401 until the DaemonSet
+# rolls. Set agent.auth.token to render offline safely.
+#
+# Not guarded with `fail`, because `lookup` is equally empty under
+# `helm install --dry-run` -- which scripts/install-nvsnap.sh --dry-run
+# relies on, and which applies nothing. helm install/upgrade is the supported
+# path and does see the existing Secret, so the token survives upgrades.
+{{-   $token = randAlphaNum 48 | b64enc }}
+{{- end }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $name }}
+  namespace: {{ $ns }}
+  labels:
+    app.kubernetes.io/name: nvsnap
+    app.kubernetes.io/part-of: nvsnap
+  annotations:
+    # helm.sh/resource-policy keeps the Secret if the release is removed with
+    # --keep-history style workflows; without it a delete/reinstall cycle
+    # silently rotates the token.
+    helm.sh/resource-policy: keep
+type: Opaque
+data:
+  token: {{ $token }}
+{{- /*
+     Replicate into every restore namespace. A SecretKeyRef resolves only in
+     the pod's own namespace, and the mount-prep init container the webhook
+     injects runs in the restore pod's namespace, not the release namespace.
+     Without a copy there its reference (marked optional, so the pod still
+     starts) silently resolves to nothing -- mount-prep then sends no header
+     and the agent 401s every restore once auth.mode=required.
+
+     Same list the restore-pod NetworkPolicy uses, so a namespace already
+     declared for egress is covered here too with no new configuration.
+*/}}
+{{- range $rns := .Values.agent.l2.restoreNamespaces }}
+{{- if ne $rns $ns }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $name }}
+  namespace: {{ $rns }}
+  labels:
+    app.kubernetes.io/name: nvsnap
+    app.kubernetes.io/part-of: nvsnap
+  annotations:
+    helm.sh/resource-policy: keep
+type: Opaque
+data:
+  token: {{ $token }}
+{{- end }}
+{{- end }}
+{{- end }}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/network-policy-restore-pods.yaml

@@ -108,7 +108,7 @@ Only rendered when the init-container strategy is selected AND
 agentHostCIDR is set. The default inline strategy does the mount inside
 the webhook and needs no pod->agent egress.
 */ -}}
-{{- if and .Values.webhook.enabled (eq (.Values.webhook.restorePrepStrategy | default "inline") "init-container") .Values.webhook.agentHostCIDR .Values.agent.l2.restoreNamespaces }}
+{{- if and .Values.webhook.enabled (eq (.Values.webhook.restorePrepStrategy | default "inline") "init-container") (or (not .Values.agent.hostNetwork) .Values.webhook.agentHostCIDR) .Values.agent.l2.restoreNamespaces }}
 {{- range $ns := .Values.agent.l2.restoreNamespaces }}
 ---
 apiVersion: networking.k8s.io/v1
@@ -130,8 +130,25 @@ spec:
     - Egress
   egress:
     - to:
+        {{- if $.Values.agent.hostNetwork }}
+        # hostNetwork: the agent carries NODE identity, so a podSelector never
+        # matches it (verified on GKE Dataplane V2 / Cilium) and the rule has
+        # to name the whole node CIDR -- every node, on this port, for every
+        # pod in the namespace.
         - ipBlock:
             cidr: {{ $.Values.webhook.agentHostCIDR }}
+        {{- else }}
+        # Pod networking: the agent has a pod identity again, so the rule can
+        # name exactly the agent pods and nothing else. This is the concrete
+        # payoff of GH #490 -- no operator-supplied CIDR, and the grant shrinks
+        # from "the node network" to "these pods".
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ $.Release.Namespace }}
+          podSelector:
+            matchLabels:
+              {{- include "nvsnap.agent.selectorLabels" $ | nindent 14 }}
+        {{- end }}
       ports:
         - protocol: TCP
           port: {{ $.Values.webhook.agentHostPort | default 8081 }}

src/compute-plane-services/nvsnap/deploy/helm/nvsnap/templates/server.yaml

@@ -137,15 +137,29 @@ spec:
             - --agent-port=8081
             - --log-level=info
             - --db-path=/data/nvsnap.db
-          {{- if .Values.jaeger.enabled }}
+          {{- if or .Values.jaeger.enabled .Values.agent.auth.enabled }}
           env:
+            {{- if .Values.jaeger.enabled }}
             # OTLP/gRPC to the bundled Jaeger collector. nvsnap-server reads
             # this via internal/tracing.Init — unset means no-op. otelhttp
             # then emits one server span per API request.
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
               value: "nvsnap-jaeger-collector.{{ .Release.Namespace }}.svc.cluster.local:4317"
             - name: NVSNAP_VERSION
               value: {{ .Values.server.image.tag | quote }}
+            {{- end }}
+            {{- if .Values.agent.auth.enabled }}
+            # The server is an agent client: it drops L1 dumps during
+            # cascade delete, dispatches captures, and polls the checkpoint
+            # list. Without the token those all 401 once the agent runs with
+            # --auth-mode=required, and the delete path fails silently —
+            # 204 returned, catalog row gone, dump orphaned (nvsnap#736).
+            - name: NVSNAP_AGENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: nvsnap-agent-token
+                  key: token
+            {{- end }}
           {{- end }}
           ports:
             - containerPort: {{ .Values.server.service.port }}