diff --git a/Makefile b/Makefile index 7d0e6db..5fbf799 100644 --- a/Makefile +++ b/Makefile @@ -138,3 +138,7 @@ cloc: ## Count lines of code excluding tests (requires cloc). .PHONY: help help: ## Show available targets. @awk 'BEGIN {FS = ":.*## "; printf "Usage: make \n\nTargets:\n"} /^[a-zA-Z0-9_-]+:.*## / {printf " %-14s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +.PHONY: api-docs +api-docs: ## Regenerate docs/api.md from the API types + GOWORK=off go run github.com/elastic/crd-ref-docs@v0.2.0 --config=hack/crd-ref-docs.yaml --source-path=. --renderer=markdown --output-path=docs/api.md --max-depth=12 diff --git a/api/v1alpha1/sandbox_conversion.go b/api/v1alpha1/sandbox_conversion.go index a40fa9e..55e6518 100644 --- a/api/v1alpha1/sandbox_conversion.go +++ b/api/v1alpha1/sandbox_conversion.go @@ -54,7 +54,7 @@ func (s *Sandbox) ConvertTo(dstRaw conversion.Hub) error { state.Status.Replicas = s.Status.Replicas stateJSON, err := json.Marshal(state) if err != nil { - return fmt.Errorf("failed to marshal v1alpha1 Sandbox state: %w", err) + return fmt.Errorf("marshal v1alpha1 sandbox state: %w", err) } dst.Annotations[v1alpha1SandboxStateAnnotation] = string(stateJSON) @@ -84,7 +84,7 @@ func (s *Sandbox) ConvertFrom(srcRaw conversion.Hub) error { var original v1alpha1State if err := json.Unmarshal([]byte(stateJSON), &original); err != nil { - return fmt.Errorf("failed to unmarshal v1alpha1 Sandbox state: %w", err) + return fmt.Errorf("unmarshal v1alpha1 sandbox state: %w", err) } // Restore replicas field from original if OperatingMode matches original intent @@ -119,7 +119,6 @@ func ConvertSpecTo(src *SandboxSpec, dst *v1beta1.SandboxSpec) { ConvertLifecycleTo(&src.Lifecycle, &dst.Lifecycle) - // Replicas -> OperatingMode if src.Replicas != nil && *src.Replicas == 0 { dst.OperatingMode = v1beta1.SandboxOperatingModeSuspended } else { @@ -143,7 +142,6 @@ func ConvertSpecFrom(src *v1beta1.SandboxSpec, dst *SandboxSpec) { ConvertLifecycleFrom(&src.Lifecycle, &dst.Lifecycle) - // OperatingMode -> Replicas if src.OperatingMode == v1beta1.SandboxOperatingModeSuspended { dst.Replicas = new(int32(0)) } else { diff --git a/api/v1alpha1/sandbox_conversion_bench_test.go b/api/v1alpha1/sandbox_conversion_bench_test.go index 129b102..595ff41 100644 --- a/api/v1alpha1/sandbox_conversion_bench_test.go +++ b/api/v1alpha1/sandbox_conversion_bench_test.go @@ -10,10 +10,6 @@ import ( v1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" ) -// BenchmarkConvertRoundTrip converts a realistically sized v1alpha1 Sandbox to -// the hub and back — the per-object cost every v1alpha1 read/write pays in the -// conversion webhook, N times per LIST. bytes/ann is the round-trip annotation -// the object carries afterwards. func BenchmarkConvertRoundTrip(b *testing.B) { src := benchSandbox() b.ReportAllocs() diff --git a/api/v1alpha1/sandbox_conversion_test.go b/api/v1alpha1/sandbox_conversion_test.go index 9608457..d0eb3be 100644 --- a/api/v1alpha1/sandbox_conversion_test.go +++ b/api/v1alpha1/sandbox_conversion_test.go @@ -64,7 +64,6 @@ func TestSandboxConversion(t *testing.T) { policy := ShutdownPolicyDelete bTrue := true - // Create src v1alpha1 Sandbox src := &Sandbox{ Name: "my-sandbox", Namespace: "default", @@ -118,13 +117,11 @@ func TestSandboxConversion(t *testing.T) { }, } - // Convert to Hub (v1beta1) dst := &v1beta1.Sandbox{} if err := src.ConvertTo(dst); err != nil { t.Fatalf("failed to convert to v1beta1: %v", err) } - // Verify src annotations and labels were not mutated during ConvertTo if val, ok := src.Annotations[v1alpha1SandboxStateAnnotation]; !ok || val != "some-old-state" { t.Errorf("src.Annotations was mutated during ConvertTo! expected 'some-old-state', got %q", val) } @@ -135,7 +132,6 @@ func TestSandboxConversion(t *testing.T) { t.Errorf("expected 1 label in src, got %d", len(src.Labels)) } - // Verify the marshaled state in dst does not contain the state annotation itself (no nesting) marshaledState := dst.Annotations[v1alpha1SandboxStateAnnotation] var stateObj Sandbox if err := json.Unmarshal([]byte(marshaledState), &stateObj); err != nil { @@ -145,7 +141,6 @@ func TestSandboxConversion(t *testing.T) { t.Errorf("dst.Annotations state nestedly contains the state annotation! causing exponential growth") } - // Verify fields in v1beta1 if dst.Spec.OperatingMode != tc.expectedMode { t.Errorf("expected OperatingMode %q, got %q", tc.expectedMode, dst.Spec.OperatingMode) } @@ -159,18 +154,15 @@ func TestSandboxConversion(t *testing.T) { t.Errorf("expected ShutdownPolicy %q, got %v", ShutdownPolicyDelete, dst.Spec.ShutdownPolicy) } - // Convert back to Spoke (v1alpha1) roundTrip := &Sandbox{} if err := roundTrip.ConvertFrom(dst); err != nil { t.Fatalf("failed to convert back to v1alpha1: %v", err) } - // Verify state annotation was stripped during ConvertFrom if _, ok := roundTrip.Annotations[v1alpha1SandboxStateAnnotation]; ok { t.Errorf("roundTrip.Annotations still contains the state annotation after ConvertFrom!") } - // Verify round-trip preserves fields losslessly if tc.replicas == nil { if roundTrip.Spec.Replicas != nil { t.Errorf("roundtrip Replicas mismatch: expected nil, got %v", *roundTrip.Spec.Replicas) @@ -200,9 +192,6 @@ func TestSandboxConversion(t *testing.T) { } } -// TestConvertFromLegacyFullObjectState pins backward compatibility: objects -// written before the slim round-trip payload carry a full v1alpha1 Sandbox -// JSON under the state annotation, and its replica fields must still restore. func TestConvertFromLegacyFullObjectState(t *testing.T) { five := int32(5) legacy := &Sandbox{ @@ -232,7 +221,6 @@ func TestConvertFromLegacyFullObjectState(t *testing.T) { } func TestSandboxConversionFromHub(t *testing.T) { - // Test conversion of a v1beta1 Sandbox created without v1alpha1 state annotation (e.g. created directly via v1beta1 API) tests := []struct { name string mode v1beta1.SandboxOperatingMode diff --git a/api/v1alpha1/sandbox_types.go b/api/v1alpha1/sandbox_types.go index 90b738e..fca09ce 100644 --- a/api/v1alpha1/sandbox_types.go +++ b/api/v1alpha1/sandbox_types.go @@ -53,8 +53,6 @@ const ( SandboxPodNameAnnotation = "agents.x-k8s.io/pod-name" // SandboxTemplateRefAnnotation is the annotation used to track the sandbox template ref. SandboxTemplateRefAnnotation = "agents.x-k8s.io/sandbox-template-ref" - // SandboxPodTemplateHashLabel is the label used to track the pod template hash. - SandboxPodTemplateHashLabel = "agents.x-k8s.io/sandbox-pod-template-hash" // SandboxPropagatedLabelsAnnotation is the annotation used to track the labels explicitly propagated from sandbox spec to pod. SandboxPropagatedLabelsAnnotation = "agents.x-k8s.io/propagated-labels" // SandboxPropagatedAnnotationsAnnotation is the annotation used to track the annotations explicitly propagated from sandbox spec to pod. diff --git a/cmd/sandbox-apiserver/main.go b/cmd/sandbox-apiserver/main.go index 05ccd43..e67b11a 100644 --- a/cmd/sandbox-apiserver/main.go +++ b/cmd/sandbox-apiserver/main.go @@ -183,7 +183,7 @@ func (o *options) serverConfig() (*genericapiserver.Config, error) { // apiserver's inventory cache: the warm-pool driver and the e2b REST surface. func (o *options) startSidecars(ctx context.Context, restCfg *restclient.Config, token string, store scale.SandboxStore, invSource scale.InventorySource) error { if o.WarmPoolDriver { - if err := startWarmPoolDriver(ctx, restCfg, token, o.WarmPoolInterval); err != nil { + if err := startWarmPoolDriver(ctx, restCfg, token, o.WarmPoolInterval, invSource); err != nil { return err } } @@ -294,8 +294,10 @@ func startInventoryCache(ctx context.Context, restCfg *restclient.Config) (cache // a poll tick (the only latency that ever mattered — the node side fills a pool // in under a second). Leader election makes exactly one of the apiserver replicas // drive the pools. The manager's own metrics/health servers are disabled; the -// aggregated apiserver owns the serving port. -func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token string, interval time.Duration) error { +// aggregated apiserver owns the serving port. inv is the process-wide cache-fed +// inventory source; the manager's own client would read NodeInventory +// unstructured and so bypass its cache on every node read. +func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token string, interval time.Duration, inv scale.InventorySource) error { scheme := runtime.NewScheme() if err := extv1beta1.AddToScheme(scheme); err != nil { return fmt.Errorf("register extensions scheme: %w", err) @@ -311,7 +313,7 @@ func startWarmPoolDriver(ctx context.Context, restCfg *restclient.Config, token if err != nil { return fmt.Errorf("build warm-pool manager: %w", err) } - driver := warmpool.New(nil, nil, token, warmpool.NewSandboxdFactory(), warmpool.Options{ + driver := warmpool.New(nil, inv, token, warmpool.NewSandboxdFactory(), warmpool.Options{ Interval: interval, Log: ctrl.Log.WithName("warmpool"), }) diff --git a/cmd/sandbox-loadgen/main.go b/cmd/sandbox-loadgen/main.go index f473f42..44cd2c4 100644 --- a/cmd/sandbox-loadgen/main.go +++ b/cmd/sandbox-loadgen/main.go @@ -37,16 +37,20 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" + asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/pkg/podruntime" ) const ( - runtimeAnnotation = "sandbox.cocoonstack.io/runtime" + runtimeAnnotation = podruntime.RuntimeAnnotation templateName = "loadgen-tpl" - warmPoolName = "loadgen-pool" + // releaseTimeout bounds the detached release so shutdown cannot hang on it. + releaseTimeout = 10 * time.Second + warmPoolName = "loadgen-pool" // observedAtAnnotation is read by the operator to compute the claim-startup // latency (time.Since). Stamping it at create-time makes the operator record // agent_sandbox_claim_startup_latency_ms for our claims. - observedAtAnnotation = "agents.x-k8s.io/controller-first-observed-at" + observedAtAnnotation = asmetrics.ObservabilityAnnotation ) var ( @@ -293,6 +297,19 @@ func (l *loadgen) claimOnce(ctx context.Context, name string) { return } claimsTotal.Inc() + // Deferred and detached from ctx: an early return on shutdown would otherwise + // strand the claim, and with it the warm sandbox, until its own expiry. + defer func() { + rel := &unstructured.Unstructured{} + rel.SetGroupVersionKind(gvk("SandboxClaim")) + rel.SetNamespace(l.o.namespace) + rel.SetName(name) + relCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout) + defer cancel() + if err := l.cl.Delete(relCtx, rel); err != nil && !apierrors.IsNotFound(err) { + claimFailed.WithLabelValues("delete").Inc() + } + }() poll := time.NewTicker(l.o.poll) defer poll.Stop() @@ -320,14 +337,6 @@ func (l *loadgen) claimOnce(ctx context.Context, name string) { break } } - // release - rel := &unstructured.Unstructured{} - rel.SetGroupVersionKind(gvk("SandboxClaim")) - rel.SetNamespace(l.o.namespace) - rel.SetName(name) - if err := l.cl.Delete(ctx, rel); err != nil && !apierrors.IsNotFound(err) { - claimFailed.WithLabelValues("delete").Inc() - } } func (l *loadgen) poolPollLoop(ctx context.Context) { diff --git a/cmd/sandbox-operator/main.go b/cmd/sandbox-operator/main.go index 7aae742..c210b44 100644 --- a/cmd/sandbox-operator/main.go +++ b/cmd/sandbox-operator/main.go @@ -27,16 +27,14 @@ import ( "strings" "time" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - "github.com/felixge/fgprof" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiruntime "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + _ "k8s.io/client-go/plugin/pkg/client/auth" // every kubeconfig auth plugin an operator may be handed "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -48,8 +46,8 @@ import ( extensionsv1alpha1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1alpha1" extensionsv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" extensionscontrollers "github.com/cocoonstack/sandbox-operator/extensions/controllers" - "github.com/cocoonstack/sandbox-operator/extensions/controllers/queue" asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/internal/queue" "github.com/cocoonstack/sandbox-operator/internal/version" "github.com/cocoonstack/sandbox-operator/pkg/podruntime" //+kubebuilder:scaffold:imports @@ -188,8 +186,7 @@ func (o *options) run() error { } defer cleanup() - // Importing net/http/pprof registers handlers on the global DefaultServeMux. - // Reset it so no server using the default mux exposes pprof by accident. + // net/http/pprof registers on DefaultServeMux at import time, so reset it. http.DefaultServeMux = http.NewServeMux() scheme := controllers.Scheme @@ -207,8 +204,13 @@ func (o *options) run() error { return fmt.Errorf("webhook certificate setup: %w", err) } + cacheByObject, err := controllers.CacheByObject() + if err != nil { + return err + } mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ Scheme: scheme, + Cache: cache.Options{ByObject: cacheByObject}, Metrics: metricsserver.Options{BindAddress: o.metricsAddr, ExtraHandlers: o.pprofHandlers()}, HealthProbeBindAddress: o.probeAddr, LeaderElection: o.enableLeaderElection, diff --git a/cmd/sandbox-operator/tls.go b/cmd/sandbox-operator/tls.go index 8647d49..b4f84a4 100644 --- a/cmd/sandbox-operator/tls.go +++ b/cmd/sandbox-operator/tls.go @@ -56,7 +56,7 @@ func generateWebhookCerts(ctx context.Context, c client.Client, certDir string, return adoptSecretCerts(secret, certDir) } if !errors.IsNotFound(getErr) { - return nil, fmt.Errorf("failed to check for existing shared Secret: %w", getErr) + return nil, fmt.Errorf("check for existing shared Secret: %w", getErr) } setupLog.Info("No shared webhook certificates found; generating new ones") @@ -65,7 +65,7 @@ func generateWebhookCerts(ctx context.Context, c client.Client, certDir string, return nil, err } if err := writeCertFiles(certDir, serverPEM, serverKeyPEM); err != nil { - return nil, fmt.Errorf("failed to write certificate files locally: %w", err) + return nil, fmt.Errorf("write certificate files locally: %w", err) } return publishSharedSecret(ctx, c, namespace, certDir, caPEM, serverPEM, serverKeyPEM) } @@ -77,7 +77,7 @@ func adoptSecretCerts(secret *corev1.Secret, certDir string) ([]byte, error) { return nil, fmt.Errorf("shared Secret %s has invalid certificate data: %w", webhookSecretName, err) } if err := writeCertFiles(certDir, serverPEM, serverKeyPEM); err != nil { - return nil, fmt.Errorf("failed to write certificate files locally: %w", err) + return nil, fmt.Errorf("write certificate files locally: %w", err) } return caPEM, nil } @@ -86,7 +86,7 @@ func adoptSecretCerts(secret *corev1.Secret, certDir string) ([]byte, error) { func issueSelfSignedPair(serviceName, namespace, clusterDomain string) (caPEM, serverPEM, serverKeyPEM []byte, err error) { caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to generate CA private key: %w", err) + return nil, nil, nil, fmt.Errorf("generate CA private key: %w", err) } caTemplate := &x509.Certificate{ SerialNumber: big.NewInt(1), @@ -99,13 +99,13 @@ func issueSelfSignedPair(serviceName, namespace, clusterDomain string) (caPEM, s } caBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create CA certificate: %w", err) + return nil, nil, nil, fmt.Errorf("create CA certificate: %w", err) } caPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caBytes}) serverKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to generate server private key: %w", err) + return nil, nil, nil, fmt.Errorf("generate server private key: %w", err) } serverTemplate := &x509.Certificate{ SerialNumber: big.NewInt(2), @@ -124,13 +124,13 @@ func issueSelfSignedPair(serviceName, namespace, clusterDomain string) (caPEM, s } serverBytes, err := x509.CreateCertificate(rand.Reader, serverTemplate, caTemplate, &serverKey.PublicKey, caKey) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create server certificate: %w", err) + return nil, nil, nil, fmt.Errorf("create server certificate: %w", err) } serverPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverBytes}) serverKeyBytes, err := x509.MarshalECPrivateKey(serverKey) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to marshal server private key: %w", err) + return nil, nil, nil, fmt.Errorf("marshal server private key: %w", err) } return caPEM, serverPEM, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: serverKeyBytes}), nil } @@ -152,13 +152,13 @@ func publishSharedSecret(ctx context.Context, c client.Client, namespace, certDi return caPEM, nil } if !errors.IsAlreadyExists(err) { - return nil, fmt.Errorf("failed to create shared Secret: %w", err) + return nil, fmt.Errorf("create shared Secret: %w", err) } setupLog.Info("Shared Secret was created concurrently by another replica; loading it", "secret", webhookSecretName) winner := &corev1.Secret{} if err := c.Get(ctx, types.NamespacedName{Name: webhookSecretName, Namespace: namespace}, winner); err != nil { - return nil, fmt.Errorf("failed to get concurrently created Secret: %w", err) + return nil, fmt.Errorf("get concurrently created Secret: %w", err) } return adoptSecretCerts(winner, certDir) } @@ -224,7 +224,7 @@ func patchCRDs(ctx context.Context, c client.Client, caPEM []byte, serviceName, setupLog.Info("CRD not found, skipping patch", "crd", name) continue } - return fmt.Errorf("failed to get CRD %s: %w", name, err) + return fmt.Errorf("get CRD %s: %w", name, err) } if crd.Spec.Conversion == nil || crd.Spec.Conversion.Strategy != apiextensionsv1.WebhookConverter { @@ -232,7 +232,6 @@ func patchCRDs(ctx context.Context, c client.Client, caPEM []byte, serviceName, continue } - // Keep a copy of the original CRD for the merge patch original := crd.DeepCopy() webhook := crd.Spec.Conversion.Webhook @@ -240,7 +239,6 @@ func patchCRDs(ctx context.Context, c client.Client, caPEM []byte, serviceName, webhook = &apiextensionsv1.WebhookConversion{} } - // Ensure ConversionReviewVersions is set when missing if len(webhook.ConversionReviewVersions) == 0 { webhook.ConversionReviewVersions = []string{"v1", "v1beta1"} } @@ -253,7 +251,6 @@ func patchCRDs(ctx context.Context, c client.Client, caPEM []byte, serviceName, webhook.ClientConfig.Service = &apiextensionsv1.ServiceReference{} } - // Update service details and caBundle webhook.ClientConfig.Service.Name = serviceName webhook.ClientConfig.Service.Namespace = namespace path := "/convert" @@ -264,7 +261,7 @@ func patchCRDs(ctx context.Context, c client.Client, caPEM []byte, serviceName, // Use Patch with MergeFrom to avoid write conflicts and managedFields issues if err := c.Patch(ctx, crd, client.MergeFrom(original)); err != nil { - return fmt.Errorf("failed to patch CRD %s: %w", name, err) + return fmt.Errorf("patch CRD %s: %w", name, err) } setupLog.Info("Successfully patched CRD with webhook configuration", "crd", name) diff --git a/cmd/sandbox-operator/tls_test.go b/cmd/sandbox-operator/tls_test.go index 007a5e7..e011792 100644 --- a/cmd/sandbox-operator/tls_test.go +++ b/cmd/sandbox-operator/tls_test.go @@ -51,13 +51,11 @@ func TestGenerateWebhookCerts(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, caPEM) - // 1. Verify files are written locally certPath := filepath.Join(tempDir, "tls.crt") keyPath := filepath.Join(tempDir, "tls.key") assert.FileExists(t, certPath) assert.FileExists(t, keyPath) - // 2. Verify server certificate has correct DNS SANs certBytes, err := os.ReadFile(certPath) require.NoError(t, err) certBlock, _ := pem.Decode(certBytes) @@ -73,7 +71,6 @@ func TestGenerateWebhookCerts(t *testing.T) { } assert.ElementsMatch(t, expectedDNSNames, cert.DNSNames) - // 3. Verify the Secret was created in the cluster secret := &corev1.Secret{} err = fakeClient.Get(t.Context(), types.NamespacedName{Name: secretName, Namespace: namespace}, secret) require.NoError(t, err) @@ -87,7 +84,6 @@ func TestGenerateWebhookCerts(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(tempDir) - // Pre-populate Secret with dummy data existingCA := []byte("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----") existingCert := []byte("-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----") existingKey := []byte("-----BEGIN EC PRIVATE KEY-----\nMIIB\n-----END EC PRIVATE KEY-----") @@ -108,7 +104,6 @@ func TestGenerateWebhookCerts(t *testing.T) { require.NoError(t, err) assert.Equal(t, existingCA, caPEM) - // Verify files are written locally with the pre-populated values certPath := filepath.Join(tempDir, "tls.crt") keyPath := filepath.Join(tempDir, "tls.key") @@ -126,7 +121,6 @@ func TestGenerateWebhookCerts(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(tempDir) - // Pre-populate Secret with invalid PEM data secret := &corev1.Secret{ Name: secretName, Namespace: namespace, @@ -151,7 +145,6 @@ func TestPatchCRDs(t *testing.T) { err := apiextensionsv1.AddToScheme(scheme) require.NoError(t, err) - // Create a helper function to build a fake CRD makeCRD := func(name string, hasWebhook bool) *apiextensionsv1.CustomResourceDefinition { crd := &apiextensionsv1.CustomResourceDefinition{ Name: name, @@ -200,8 +193,8 @@ func TestPatchCRDs(t *testing.T) { t.Run("successfully patches CRDs with Webhook strategy", func(t *testing.T) { crd1 := makeCRD("sandboxes.agents.x-k8s.io", true) crd2 := makeCRD("sandboxclaims.extensions.agents.x-k8s.io", true) - // crd3 is not installed (simulating extensions disabled) - crd4 := makeCRD("sandboxwarmpools.extensions.agents.x-k8s.io", false) // has None strategy + + crd4 := makeCRD("sandboxwarmpools.extensions.agents.x-k8s.io", false) fakeClient := fake.NewClientBuilder(). WithScheme(scheme). @@ -215,7 +208,6 @@ func TestPatchCRDs(t *testing.T) { err := patchCRDs(t.Context(), fakeClient, caPEM, serviceName, namespace, true) require.NoError(t, err) - // Verify crd1 was patched patchedCRD1 := &apiextensionsv1.CustomResourceDefinition{} err = fakeClient.Get(t.Context(), types.NamespacedName{Name: "sandboxes.agents.x-k8s.io"}, patchedCRD1) require.NoError(t, err) @@ -226,14 +218,12 @@ func TestPatchCRDs(t *testing.T) { assert.Equal(t, "/convert", *patchedCRD1.Spec.Conversion.Webhook.ClientConfig.Service.Path) assert.Equal(t, caPEM, patchedCRD1.Spec.Conversion.Webhook.ClientConfig.CABundle) - // Verify crd2 was patched patchedCRD2 := &apiextensionsv1.CustomResourceDefinition{} err = fakeClient.Get(t.Context(), types.NamespacedName{Name: "sandboxclaims.extensions.agents.x-k8s.io"}, patchedCRD2) require.NoError(t, err) assert.Equal(t, serviceName, patchedCRD2.Spec.Conversion.Webhook.ClientConfig.Service.Name) assert.Equal(t, caPEM, patchedCRD2.Spec.Conversion.Webhook.ClientConfig.CABundle) - // Verify crd4 was NOT patched (strategy remains None) patchedCRD4 := &apiextensionsv1.CustomResourceDefinition{} err = fakeClient.Get(t.Context(), types.NamespacedName{Name: "sandboxwarmpools.extensions.agents.x-k8s.io"}, patchedCRD4) require.NoError(t, err) @@ -242,9 +232,6 @@ func TestPatchCRDs(t *testing.T) { }) t.Run("skips extension CRDs when extensions disabled", func(t *testing.T) { - // With --extensions=false the extension conversion webhooks are never - // registered, so their CRD caBundles must not be patched — otherwise the - // apiserver routes conversion to an endpoint this process does not serve. crd1 := makeCRD("sandboxes.agents.x-k8s.io", true) crd2 := makeCRD("sandboxclaims.extensions.agents.x-k8s.io", true) @@ -256,14 +243,11 @@ func TestPatchCRDs(t *testing.T) { err := patchCRDs(t.Context(), fakeClient, []byte("ca"), "svc", "ns", false) require.NoError(t, err) - // The core Sandbox CRD is still patched. patchedCRD1 := &apiextensionsv1.CustomResourceDefinition{} require.NoError(t, fakeClient.Get(t.Context(), types.NamespacedName{Name: "sandboxes.agents.x-k8s.io"}, patchedCRD1)) require.NotNil(t, patchedCRD1.Spec.Conversion.Webhook) assert.Equal(t, "svc", patchedCRD1.Spec.Conversion.Webhook.ClientConfig.Service.Name) - // The extension CRD is left untouched: its original caBundle and service - // are unchanged (not overwritten with the new values). untouchedCRD2 := &apiextensionsv1.CustomResourceDefinition{} require.NoError(t, fakeClient.Get(t.Context(), types.NamespacedName{Name: "sandboxclaims.extensions.agents.x-k8s.io"}, untouchedCRD2)) assert.Equal(t, []byte("old-ca"), untouchedCRD2.Spec.Conversion.Webhook.ClientConfig.CABundle) diff --git a/cmd/sandbox-sdk-loadgen/main.go b/cmd/sandbox-sdk-loadgen/main.go index 80feffd..b7c9a08 100644 --- a/cmd/sandbox-sdk-loadgen/main.go +++ b/cmd/sandbox-sdk-loadgen/main.go @@ -52,6 +52,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" + "github.com/cocoonstack/sandbox-operator/pkg/scale" + "github.com/cocoonstack/sandbox-operator/pkg/scale/apiserver" sdk "github.com/cocoonstack/sandbox/sdk/go" ) @@ -59,9 +61,13 @@ import ( // address, per-sandbox exec token, and sandboxd claim id. Together they let the // loadgen exec into exactly what it claimed (create -> exec latency). const ( - addressAnnotation = "sandbox.cocoonstack.io/address" - tokenAnnotation = "sandbox.cocoonstack.io/token" - claimIDAnnotation = "sandbox.cocoonstack.io/claim-id" + addressAnnotation = apiserver.AddressAnnotation + tokenAnnotation = apiserver.TokenAnnotation + claimIDAnnotation = scale.ClaimIDAnnotation + + // metricsReadHeaderTimeout bounds how long a client may take to send its + // request headers, so a stalled connection cannot pin a handler. + metricsReadHeaderTimeout = 5 * time.Second ) var ( @@ -245,7 +251,12 @@ func main() { mux := http.NewServeMux() mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("ok")) }) - if err := http.ListenAndServe(o.metricsAddr, mux); err != nil { //nolint:gosec // internal metrics endpoint + srv := &http.Server{ + Addr: o.metricsAddr, + Handler: mux, + ReadHeaderTimeout: metricsReadHeaderTimeout, + } + if err := srv.ListenAndServe(); err != nil { fatalf("metrics server: %v", err) } }() @@ -513,7 +524,7 @@ func newSandbox(ns, name, image string) *sandboxv1beta1.Sandbox { return &sandboxv1beta1.Sandbox{ Namespace: ns, Name: name, - Labels: map[string]string{"agents.x-k8s.io/created-by": "sdk-loadgen"}, + Labels: map[string]string{sandboxv1beta1.CreatedByLabel: "sdk-loadgen"}, Spec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{ PodTemplate: sandboxv1beta1.PodTemplate{ diff --git a/cmd/sandbox-sdk-loadgen/main_test.go b/cmd/sandbox-sdk-loadgen/main_test.go index 0e4ca79..1d88bbb 100644 --- a/cmd/sandbox-sdk-loadgen/main_test.go +++ b/cmd/sandbox-sdk-loadgen/main_test.go @@ -12,11 +12,6 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" ) -// TestRunIssuesExactlyTotal encodes the safety property that caused the -// 2026-07-21 incident when it was absent: a run must issue EXACTLY --total -// creates across all workers — never more — and with cleanup on, every created -// sandbox must be released (leaked == 0). If this test fails, the loadgen can -// again run away and drain the fleet's warm pools. func TestRunIssuesExactlyTotal(t *testing.T) { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -57,8 +52,6 @@ func TestRunIssuesExactlyTotal(t *testing.T) { } } -// TestConcurrencyClampedToTotal guards the small-run path: asking for 2 creates -// with 10 workers must still issue exactly 2. func TestConcurrencyClampedToTotal(t *testing.T) { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) diff --git a/controllers/pod_metadata_test.go b/controllers/pod_metadata_test.go index 7ae8518..0bc0932 100644 --- a/controllers/pod_metadata_test.go +++ b/controllers/pod_metadata_test.go @@ -133,8 +133,6 @@ func TestUpdatePodMetadata(t *testing.T) { } func TestResourceOwnershipIotaValues(t *testing.T) { - // The iota block was merged into a larger const block; these must stay 0/1/2 - // because checkOwnership's switch and every caller depend on the ordering. for _, c := range []struct { got resourceOwnership want int diff --git a/controllers/sandbox_controller.go b/controllers/sandbox_controller.go index f404adb..f52b549 100644 --- a/controllers/sandbox_controller.go +++ b/controllers/sandbox_controller.go @@ -28,6 +28,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -35,6 +36,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/log" @@ -132,7 +134,6 @@ func (r *SandboxReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - // Initialize trace ID for active resources missing an ID (inline, no re-reconcile) tc := r.Tracer.GetTraceContext(ctx) if tc != "" && (sandbox.Annotations == nil || sandbox.Annotations[asmetrics.TraceContextAnnotation] == "") { patch := client.MergeFrom(sandbox.DeepCopy()) @@ -184,7 +185,7 @@ func (r *SandboxReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct func (r *SandboxReconciler) SetupWithManager(mgr ctrl.Manager, concurrentWorkers int) error { if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, podSandboxNameHashIndex, podSandboxNameHashIndexer); err != nil { - return fmt.Errorf("failed to index pods by sandbox label: %w", err) + return fmt.Errorf("index pods by sandbox label: %w", err) } labelSelectorPredicate, err := predicate.LabelSelectorPredicate(metav1.LabelSelector{ @@ -227,11 +228,9 @@ func (r *SandboxReconciler) reconcileChildResources(ctx context.Context, sandbox sandbox.Status.NodeName = pod.Spec.NodeName } - // Reconcile Service svc, err := r.reconcileService(ctx, sandbox, nameHash) allErrors = errors.Join(allErrors, err) - // compute and set overall conditions conditions := r.computeConditions(sandbox, allErrors, svc, pod) hasFinished := false for _, condition := range conditions { @@ -275,7 +274,6 @@ func (r *SandboxReconciler) computeSuspendedCondition(sandbox *sandboxv1beta1.Sa ObservedGeneration: sandbox.Generation, } if pod == nil { - // Mark Suspended condition as True suspended.Status = metav1.ConditionTrue suspended.Reason = sandboxv1beta1.SandboxReasonSuspendedPodTerminated suspended.Message = "Pod has been terminated. Sandbox is not operational." @@ -396,7 +394,6 @@ func (r *SandboxReconciler) updateStatus(ctx context.Context, oldStatus *sandbox return err } - // Surface error return nil } @@ -418,25 +415,22 @@ func (r *SandboxReconciler) reconcileService(ctx context.Context, sandbox *sandb return r.createHeadlessService(ctx, sandbox, nameHash) } - // Service exists logger.Info("Found Service", "Service.Namespace", service.Namespace, "Service.Name", service.Name) ownership, controllerRef := checkOwnership(service, sandbox) if desired != nil && !*desired { - // desired is false — delete owned service if ownership == resourceOwnedBySandbox { logger.Info("Deleting owned service because service is disabled", "Service.Name", service.Name, "Sandbox.Name", sandbox.Name) if err := r.Delete(ctx, service); err != nil && !k8serrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to delete service: %w", err) + return nil, fmt.Errorf("delete service: %w", err) } } r.clearServiceStatus(sandbox) return nil, nil } - // desired == nil or true switch ownership { case resourceOwnedByOther: logger.Info("Refusing to use service: service is owned by a different controller", @@ -447,11 +441,9 @@ func (r *SandboxReconciler) reconcileService(ctx context.Context, sandbox *sandb case resourceUnowned: if desired == nil { - // desired is nil + unowned service — do not adopt r.clearServiceStatus(sandbox) return nil, nil } - // desired is true + unowned service — adopt isAdoptablePool := isAdoptable(service) hasTrackingLabel := service.Labels != nil && service.Labels[sandboxLabel] == nameHash if !isAdoptablePool && !hasTrackingLabel { @@ -483,7 +475,7 @@ func (r *SandboxReconciler) reconcileService(ctx context.Context, sandbox *sandb return nil, fmt.Errorf("SetControllerReference for Service failed: %w", err) } if err := r.Update(ctx, service); err != nil { - return nil, fmt.Errorf("failed to update service with owner reference: %w", err) + return nil, fmt.Errorf("update service with owner reference: %w", err) } case resourceOwnedBySandbox: @@ -508,7 +500,7 @@ func (r *SandboxReconciler) reconcileService(ctx context.Context, sandbox *sandb if needsUpdate { logger.Info("Reconciling owned service drift", "Service.Namespace", service.Namespace, "Service.Name", service.Name, "Sandbox.Namespace", sandbox.Namespace, "Sandbox.Name", sandbox.Name) if err := r.Patch(ctx, service, patch); err != nil { - return nil, fmt.Errorf("failed to patch owned service: %w", err) + return nil, fmt.Errorf("patch owned service: %w", err) } } } @@ -526,7 +518,7 @@ func (r *SandboxReconciler) clearPodNameAnnotation(ctx context.Context, sandbox patch := client.MergeFrom(sandbox.DeepCopy()) delete(sandbox.Annotations, sandboxv1beta1.SandboxPodNameAnnotation) if err := r.Patch(ctx, sandbox, patch); err != nil { - return fmt.Errorf("failed to clear pod name annotation: %w", err) + return fmt.Errorf("clear pod name annotation: %w", err) } logger.Info("Removed pod name annotation from sandbox", "Sandbox.Name", sandbox.Name) return nil @@ -575,11 +567,12 @@ func (r *SandboxReconciler) reconcilePod(ctx context.Context, sandbox *sandboxv1 ctx, end := r.Tracer.StartSpan(ctx, nil, "reconcilePod", nil) defer end() - // TODO: find a better way to make sure one sandbox has at most one pod + // Only the count is read, so the cache may hand back its own objects. podList := &corev1.PodList{} if err := r.List(ctx, podList, client.InNamespace(sandbox.Namespace), client.MatchingFields{podSandboxNameHashIndex: nameHash}, + client.UnsafeDisableDeepCopy, ); err != nil { logger.Error(err, "Failed to list pods") return nil, fmt.Errorf("pod list failed: %w", err) @@ -643,7 +636,7 @@ func (r *SandboxReconciler) suspendPod(ctx context.Context, sandbox *sandboxv1be } logger.Info("Deleting Pod because .Spec.OperatingMode is Suspended", "Pod.Namespace", pod.Namespace, "Pod.Name", pod.Name) if err := r.Delete(ctx, pod); err != nil { - return fmt.Errorf("failed to delete pod: %w", err) + return fmt.Errorf("delete pod: %w", err) } case resourceUnowned: logger.Info("Refusing to delete pod: pod has no controllerRef pointing to this sandbox", @@ -701,13 +694,12 @@ func (r *SandboxReconciler) adoptPod(ctx context.Context, sandbox *sandboxv1beta if r.updatePodMetadata(ctx, pod, sandbox, nameHash) || needsUpdate { if err := r.Patch(ctx, pod, patch); err != nil { - return nil, fmt.Errorf("failed to patch pod: %w", err) + return nil, fmt.Errorf("patch pod: %w", err) } } if err := r.ensurePodNameAnnotation(ctx, sandbox, pod.Name); err != nil { return nil, err } - // TODO - Do we enforce (change) spec if a pod exists ? return pod, nil } @@ -783,7 +775,7 @@ func (r *SandboxReconciler) createPod(ctx context.Context, sandbox *sandboxv1bet logger.Info("Pod already exists, fetching existing pod", "Pod.Namespace", pod.Namespace, "Pod.Name", pod.Name) existing := &corev1.Pod{} if getErr := r.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, existing); getErr != nil { - return nil, fmt.Errorf("pod already exists but failed to fetch: %w", getErr) + return nil, fmt.Errorf("fetch existing pod: %w", getErr) } return r.adoptPod(ctx, sandbox, existing, nameHash) } @@ -818,7 +810,7 @@ func (r *SandboxReconciler) ensurePodNameAnnotation(ctx context.Context, sandbox } sandbox.Annotations[sandboxv1beta1.SandboxPodNameAnnotation] = podName if err := r.Patch(ctx, sandbox, patch); err != nil { - return fmt.Errorf("failed to set pod name annotation: %w", err) + return fmt.Errorf("set pod name annotation: %w", err) } return nil } @@ -874,7 +866,6 @@ func (r *SandboxReconciler) updatePodMetadata(ctx context.Context, pod *corev1.P func (r *SandboxReconciler) reconcilePVCs(ctx context.Context, sandbox *sandboxv1beta1.Sandbox, nameHash string) error { logger := log.FromContext(ctx) - // Start a child span of ReconcileSandbox ctx, end := r.Tracer.StartSpan(ctx, nil, "reconcilePVCs", nil) defer end() @@ -910,7 +901,7 @@ func (r *SandboxReconciler) reconcilePVCs(ctx context.Context, sandbox *sandboxv return fmt.Errorf("SetControllerReference for PVC failed: %w", err) } if err := r.Patch(ctx, pvc, patch); err != nil { - return fmt.Errorf("failed to patch PVC with owner reference: %w", err) + return fmt.Errorf("patch PVC with owner reference: %w", err) } case resourceOwnedBySandbox: @@ -921,7 +912,7 @@ func (r *SandboxReconciler) reconcilePVCs(ctx context.Context, sandbox *sandboxv if !k8serrors.IsNotFound(getErr) { logger.Error(getErr, "Failed to get PVC") - return fmt.Errorf("failed to get PVC: %w", getErr) + return fmt.Errorf("get PVC: %w", getErr) } pvcLabels := maps.Clone(pvcTemplate.Labels) @@ -949,16 +940,15 @@ func (r *SandboxReconciler) reconcilePVCs(ctx context.Context, sandbox *sandboxv return nil } -// handles sandbox expiry by deleting child resources and the sandbox itself if needed. +// handleSandboxExpiry deletes the expired sandbox's children and, per its shutdown policy, the sandbox itself. func (r *SandboxReconciler) handleSandboxExpiry(ctx context.Context, sandbox *sandboxv1beta1.Sandbox) (bool, error) { var allErrors error - // Delete children only if owned by this sandbox podName := resolvePodName(sandbox) pod := &corev1.Pod{} if err := r.Get(ctx, types.NamespacedName{Name: podName, Namespace: sandbox.Namespace}, pod); err != nil { if !k8serrors.IsNotFound(err) { - allErrors = errors.Join(allErrors, fmt.Errorf("failed to get pod: %w", err)) + allErrors = errors.Join(allErrors, fmt.Errorf("get pod: %w", err)) } } else { allErrors = errors.Join(allErrors, r.deleteExpiredChild(ctx, sandbox, pod, "pod")) @@ -967,7 +957,7 @@ func (r *SandboxReconciler) handleSandboxExpiry(ctx context.Context, sandbox *sa service := &corev1.Service{} if err := r.Get(ctx, types.NamespacedName{Name: sandbox.Name, Namespace: sandbox.Namespace}, service); err != nil { if !k8serrors.IsNotFound(err) { - allErrors = errors.Join(allErrors, fmt.Errorf("failed to get service: %w", err)) + allErrors = errors.Join(allErrors, fmt.Errorf("get service: %w", err)) } } else { allErrors = errors.Join(allErrors, r.deleteExpiredChild(ctx, sandbox, service, "service")) @@ -975,14 +965,12 @@ func (r *SandboxReconciler) handleSandboxExpiry(ctx context.Context, sandbox *sa if sandbox.Spec.ShutdownPolicy != nil && *sandbox.Spec.ShutdownPolicy == sandboxv1beta1.ShutdownPolicyDelete { if err := r.Delete(ctx, sandbox); err != nil && !k8serrors.IsNotFound(err) { - allErrors = errors.Join(allErrors, fmt.Errorf("failed to delete sandbox: %w", err)) + allErrors = errors.Join(allErrors, fmt.Errorf("delete sandbox: %w", err)) } else { return true, nil } } - // If we reach here, sandbox is not deleted - // Only update "expired" status if cleanup was successful if allErrors == nil { // Drop live-resource status while retaining terminal conditions. conditions := sandbox.Status.Conditions @@ -1001,38 +989,40 @@ func (r *SandboxReconciler) deleteExpiredChild(ctx context.Context, sandbox *san switch ownership { case resourceOwnedBySandbox: if err := r.Delete(ctx, obj); err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to delete %s: %w", kind, err) + return fmt.Errorf("delete %s: %w", kind, err) } case resourceUnowned: - logger.Info("Skipping "+kind+" deletion during expiry: no controllerRef pointing to this sandbox", - "Name", obj.GetName(), "Sandbox.Name", sandbox.Name) + logger.Info("Skipping child deletion during expiry: no controllerRef pointing to this sandbox", + "Kind", kind, "Name", obj.GetName(), "Sandbox.Name", sandbox.Name) case resourceOwnedByOther: - logger.Info("Skipping "+kind+" deletion during expiry: owned by a different controller", - "Name", obj.GetName(), "Sandbox.Name", sandbox.Name, + logger.Info("Skipping child deletion during expiry: owned by a different controller", + "Kind", kind, "Name", obj.GetName(), "Sandbox.Name", sandbox.Name, "Owner.Kind", controllerRef.Kind, "Owner.Name", controllerRef.Name, "Owner.UID", controllerRef.UID) } return nil } -// checkOwnership determines whether a Kubernetes resource is owned by the given Sandbox, -// has no controller, or is owned by a different controller. -// It returns both the ownership classification and the controller reference (if any), -// so callers can log owner details without redundant GetControllerOf calls. -func checkOwnership(obj client.Object, sandbox *sandboxv1beta1.Sandbox) (resourceOwnership, *metav1.OwnerReference) { - controllerRef := metav1.GetControllerOf(obj) - if controllerRef == nil { - return resourceUnowned, nil - } - if controllerRef.UID == sandbox.UID { - return resourceOwnedBySandbox, controllerRef +// CacheByObject scopes a manager cache to the child objects this controller +// labels, so the Pod/Service/PVC informers never watch the whole cluster. Every +// cached read this controller makes targets a labeled child, and the PVC entry +// also stops the first volumeClaimTemplates Sandbox from spinning up a +// cluster-wide informer mid-reconcile. +func CacheByObject() (map[client.Object]cache.ByObject, error) { + sel, err := labels.Parse(sandboxLabel) + if err != nil { + return nil, fmt.Errorf("parse sandbox cache selector: %w", err) } - return resourceOwnedByOther, controllerRef + return map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: sel}, + &corev1.Service{}: {Label: sel}, + &corev1.PersistentVolumeClaim{}: {Label: sel}, + }, nil } // MergeVolumeClaimVolumes merges PVC-backed volumes into an existing volume // list, replacing any volumes with matching names. This follows StatefulSet // semantics where volumeClaimTemplate volumes take priority. -func MergeVolumeClaimVolumes(existing []corev1.Volume, pvcVolumes []corev1.Volume) []corev1.Volume { +func MergeVolumeClaimVolumes(existing, pvcVolumes []corev1.Volume) []corev1.Volume { if len(pvcVolumes) == 0 { return existing } @@ -1049,9 +1039,23 @@ func MergeVolumeClaimVolumes(existing []corev1.Volume, pvcVolumes []corev1.Volum return append(filtered, pvcVolumes...) } -// checks if the sandbox has expired -// returns true if expired, false otherwise -// if not expired, also returns the duration to requeue after. +// checkOwnership determines whether a Kubernetes resource is owned by the given Sandbox, +// has no controller, or is owned by a different controller. +// It returns both the ownership classification and the controller reference (if any), +// so callers can log owner details without redundant GetControllerOf calls. +func checkOwnership(obj client.Object, sandbox *sandboxv1beta1.Sandbox) (resourceOwnership, *metav1.OwnerReference) { + controllerRef := metav1.GetControllerOf(obj) + if controllerRef == nil { + return resourceUnowned, nil + } + if controllerRef.UID == sandbox.UID { + return resourceOwnedBySandbox, controllerRef + } + return resourceOwnedByOther, controllerRef +} + +// checkSandboxExpiry reports whether sandbox has expired and, when it has not, +// the delay to requeue after. func checkSandboxExpiry(sandbox *sandboxv1beta1.Sandbox, now time.Time) (bool, time.Duration) { if sandbox.Spec.ShutdownTime == nil { return false, 0 @@ -1060,14 +1064,7 @@ func checkSandboxExpiry(sandbox *sandboxv1beta1.Sandbox, now time.Time) (bool, t if !now.Before(shutdownTime) { return true, 0 } - remainingTime := shutdownTime.Sub(now) - - // TODO(barney-s): Do we need a inverse exponential backoff here ? - // requeueAfter := max(remainingTime/2, 2*time.Second) - - // Requeue at expiry time or in 2 seconds whichever is later - requeueAfter := max(remainingTime, 2*time.Second) - return false, requeueAfter + return false, max(shutdownTime.Sub(now), 2*time.Second) } func setSandboxExpiredCondition(sandbox *sandboxv1beta1.Sandbox) { diff --git a/controllers/sandbox_controller_test.go b/controllers/sandbox_controller_test.go index 1fb9583..c1e272f 100644 --- a/controllers/sandbox_controller_test.go +++ b/controllers/sandbox_controller_test.go @@ -281,7 +281,7 @@ func TestReconcile(t *testing.T) { }{ { name: "minimal sandbox spec creates Pod but not Service by default", - // Input sandbox spec + sandboxSpec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{ Spec: corev1.PodSpec{ @@ -293,7 +293,7 @@ func TestReconcile(t *testing.T) { }, }}, }, - // Verify Sandbox status + wantStatus: sandboxv1beta1.SandboxStatus{ LabelSelector: "agents.x-k8s.io/sandbox-name-hash=" + nameHash, Conditions: []metav1.Condition{ @@ -307,7 +307,6 @@ func TestReconcile(t *testing.T) { }, }, wantObjs: []client.Object{ - // Verify Pod &corev1.Pod{ Name: sandboxName, Namespace: sandboxNs, @@ -328,7 +327,7 @@ func TestReconcile(t *testing.T) { }, { name: "minimal sandbox spec with Pod and Service", - // Input sandbox spec + sandboxSpec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{ Service: new(true), @@ -343,7 +342,7 @@ func TestReconcile(t *testing.T) { }, }, }, - // Verify Sandbox status + wantStatus: sandboxv1beta1.SandboxStatus{ Service: sandboxName, ServiceFQDN: "sandbox-name.sandbox-ns.svc.cluster.local", @@ -359,7 +358,6 @@ func TestReconcile(t *testing.T) { }, }, wantObjs: []client.Object{ - // Verify Pod &corev1.Pod{ Name: sandboxName, Namespace: sandboxNs, @@ -376,7 +374,7 @@ func TestReconcile(t *testing.T) { }, }, }, - // Verify Service + &corev1.Service{ Name: sandboxName, Namespace: sandboxNs, @@ -396,7 +394,7 @@ func TestReconcile(t *testing.T) { }, { name: "sandbox spec with PVC, Pod, and Service", - // Input sandbox spec + sandboxSpec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{ Service: new(true), @@ -434,7 +432,7 @@ func TestReconcile(t *testing.T) { }, }, }, - // Verify Sandbox status + wantStatus: sandboxv1beta1.SandboxStatus{ Service: sandboxName, ServiceFQDN: "sandbox-name.sandbox-ns.svc.cluster.local", @@ -450,7 +448,6 @@ func TestReconcile(t *testing.T) { }, }, wantObjs: []client.Object{ - // Verify Pod &corev1.Pod{ Name: sandboxName, Namespace: sandboxNs, @@ -482,7 +479,7 @@ func TestReconcile(t *testing.T) { }, }, }, - // Verify Service + &corev1.Service{ Name: sandboxName, Namespace: sandboxNs, @@ -498,7 +495,7 @@ func TestReconcile(t *testing.T) { ClusterIP: "None", }, }, - // Verify PVC + &corev1.PersistentVolumeClaim{ Name: "my-pvc-sandbox-name", Namespace: sandboxNs, @@ -570,7 +567,6 @@ func TestReconcile(t *testing.T) { }, }, wantObjs: []client.Object{ - // Verifying Service exists (Pod was verified indirectly via state, and owner reference is added in reconcilePod test suite) &corev1.Service{ Name: sandboxName, Namespace: sandboxNs, @@ -871,7 +867,7 @@ func TestReconcile(t *testing.T) { }, }, }, - // Pod should NOT be deleted (owned by other), Service SHOULD be deleted (owned by sandbox) + wantDeletedObjs: []client.Object{ &corev1.Service{Name: sandboxName, Namespace: sandboxNs}, }, @@ -884,11 +880,8 @@ func TestReconcile(t *testing.T) { reconcileCount: 2, initialObjs: []runtime.Object{ &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxName, - Namespace: sandboxNs, - // No owner references - }, + Name: sandboxName, + Namespace: sandboxNs, }, &corev1.Service{ Name: sandboxName, @@ -984,7 +977,7 @@ func TestReconcile(t *testing.T) { }) require.NoError(t, err) } - // Validate Sandbox status or deletion + liveSandbox := &sandboxv1beta1.Sandbox{} err = r.Get(t.Context(), types.NamespacedName{Name: sandboxName, Namespace: sandboxNs}, liveSandbox) if tc.expectSandboxDeleted { @@ -998,7 +991,7 @@ func TestReconcile(t *testing.T) { t.Fatalf("unexpected sandbox status (-want,+got):\n%s", diff) } } - // Validate the other objects from the "cluster" (fake client) + for _, obj := range tc.wantObjs { liveObj := obj.DeepCopyObject().(client.Object) err = r.Get(t.Context(), types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, liveObj) @@ -1055,7 +1048,7 @@ func TestReconcilePod(t *testing.T) { wantPod *corev1.Pod expectErr bool wantSandboxAnnotations map[string]string - wantPodSurvives string // if set, verify this pod still exists after reconcile + wantPodSurvives string }{ { name: "updates label and owner reference if Pod already exists", @@ -1108,22 +1101,18 @@ func TestReconcilePod(t *testing.T) { name: "persists owner reference when adopting unowned pod whose labels are already correct", initialObjs: []runtime.Object{ &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: sandboxName, - Namespace: sandboxNs, - ResourceVersion: "1", - Labels: map[string]string{ - "agents.x-k8s.io/sandbox-name-hash": nameHash, - "custom-label": "label-val", - sandboxv1beta1.SandboxAdoptableLabel: "true", - }, - Annotations: map[string]string{ - "custom-annotation": "anno-val", - "agents.x-k8s.io/propagated-labels": "custom-label", - "agents.x-k8s.io/propagated-annotations": "custom-annotation", - }, - // No OwnerReferences : simulates a pre-created pod whose - // labels/annotations already match the sandbox spec exactly. + Name: sandboxName, + Namespace: sandboxNs, + ResourceVersion: "1", + Labels: map[string]string{ + "agents.x-k8s.io/sandbox-name-hash": nameHash, + "custom-label": "label-val", + sandboxv1beta1.SandboxAdoptableLabel: "true", + }, + Annotations: map[string]string{ + "custom-annotation": "anno-val", + "agents.x-k8s.io/propagated-labels": "custom-label", + "agents.x-k8s.io/propagated-annotations": "custom-annotation", }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{Name: "test-container"}}, @@ -1240,8 +1229,6 @@ func TestReconcilePod(t *testing.T) { }, ObjectMeta: sandboxv1beta1.PodMetadata{ Labels: map[string]string{ - // Attacker attempts to hijack another Sandbox's routing label - // and to spoof an extensions-prefixed system label. "agents.x-k8s.io/sandbox-name-hash": "malicious-hijacked-hash", "extensions.agents.x-k8s.io/warm-pool-spoof": "evil", "custom-label": "label-val", @@ -1260,7 +1247,6 @@ func TestReconcilePod(t *testing.T) { Namespace: sandboxNs, ResourceVersion: "1", Labels: map[string]string{ - // System label is set by the controller, not the attacker's value. "agents.x-k8s.io/sandbox-name-hash": nameHash, "custom-label": "label-val", }, @@ -1288,12 +1274,12 @@ func TestReconcilePod(t *testing.T) { Labels: map[string]string{ "agents.x-k8s.io/sandbox-name-hash": nameHash, "custom-label": "label-val", - // A system label an older controller propagated and recorded. + "agents.x-k8s.io/evil": "x", }, Annotations: map[string]string{ "custom-annotation": "anno-val", - // Older controller recorded system keys in the propagated lists. + "agents.x-k8s.io/propagated-labels": "custom-label,agents.x-k8s.io/evil", "agents.x-k8s.io/propagated-annotations": "custom-annotation,agents.x-k8s.io/pod-name,opentelemetry.io/trace-context", "agents.x-k8s.io/pod-name": "leftover", @@ -1877,7 +1863,7 @@ func TestReconcilePod(t *testing.T) { Name: sandboxName, Namespace: sandboxNs, ResourceVersion: "1", - // Add a controller reference to a different controller + OwnerReferences: []metav1.OwnerReference{ { APIVersion: "apps/v1", @@ -2345,7 +2331,7 @@ func TestReconcilePod(t *testing.T) { pod, err := r.reconcilePod(t.Context(), sandbox, nameHash) if tc.expectErr { require.Error(t, err) - // Verify that any initially unowned Pod remains unowned (never adopted) + for _, obj := range tc.initialObjs { if initialPod, ok := obj.(*corev1.Pod); ok { if len(initialPod.OwnerReferences) == 0 { @@ -2361,7 +2347,6 @@ func TestReconcilePod(t *testing.T) { } require.Equal(t, tc.wantPod, pod) - // Validate the Pod from the "cluster" (fake client) if tc.wantPod != nil { livePod := &corev1.Pod{} err = r.Get(t.Context(), types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, livePod) @@ -2369,12 +2354,12 @@ func TestReconcilePod(t *testing.T) { require.Equal(t, tc.wantPod, livePod) } else if !tc.expectErr { if tc.wantPodSurvives != "" { - // Pod should still exist (ownership check blocked deletion) + livePod := &corev1.Pod{} err = r.Get(t.Context(), types.NamespacedName{Name: tc.wantPodSurvives, Namespace: sandboxNs}, livePod) require.NoError(t, err, "expected pod %q to survive but it was deleted", tc.wantPodSurvives) } else { - // When wantPod is nil and no error expected, verify pod doesn't exist + livePod := &corev1.Pod{} podName := sandboxName if annotatedPod, exists := tc.sandbox.Annotations[sandboxv1beta1.SandboxPodNameAnnotation]; exists && annotatedPod != "" { @@ -2416,7 +2401,7 @@ func TestReconcileService(t *testing.T) { sandbox *sandboxv1beta1.Sandbox wantService *corev1.Service expectErr bool - errContains string // substring that must appear in the error + errContains string wantNilService bool wantServiceDeleted bool wantStatusService string @@ -2787,7 +2772,7 @@ func TestReconcileService(t *testing.T) { if tc.errContains != "" { require.Contains(t, err.Error(), tc.errContains) } - // Verify that any initially unowned Service remains unowned (never adopted) + for _, obj := range tc.initialObjs { if initialSvc, ok := obj.(*corev1.Service); ok { if len(initialSvc.OwnerReferences) == 0 { @@ -2807,13 +2792,11 @@ func TestReconcileService(t *testing.T) { } } - // Verify status was set correctly if !tc.expectErr { require.Equal(t, tc.wantStatusService, tc.sandbox.Status.Service) require.Equal(t, tc.wantStatusServiceFQDN, tc.sandbox.Status.ServiceFQDN) } - // Verify the live service in the fake client matches expected state if tc.wantService != nil { liveSvc := &corev1.Service{} err = r.Get(t.Context(), types.NamespacedName{ @@ -2936,7 +2919,7 @@ func TestReconcilePVCs(t *testing.T) { localSandboxUID := types.UID("sandbox-uid-123") otherUID := types.UID("other-uid-456") pvcTemplateName := "data" - pvcName := pvcTemplateName + "-" + sandboxName // "data-test-sandbox" + pvcName := pvcTemplateName + "-" + sandboxName nameHash := hash.Name(sandboxName) sandbox := &sandboxv1beta1.Sandbox{ @@ -3063,7 +3046,7 @@ func TestReconcilePVCs(t *testing.T) { if tc.errContains != "" { require.Contains(t, err.Error(), tc.errContains) } - // Verify that any initially unowned PVC remains unowned (never adopted) + for _, obj := range tc.initialObjs { if initialPVC, ok := obj.(*corev1.PersistentVolumeClaim); ok { if len(initialPVC.OwnerReferences) == 0 { @@ -3079,7 +3062,6 @@ func TestReconcilePVCs(t *testing.T) { require.NoError(t, err) - // Verify PVC exists and is owned by the sandbox. livePVC := &corev1.PersistentVolumeClaim{} err = r.Get(t.Context(), types.NamespacedName{Name: pvcName, Namespace: sandboxNs}, livePVC) require.NoError(t, err) @@ -3323,10 +3305,10 @@ func TestMergeVolumeClaimVolumes(t *testing.T) { result := MergeVolumeClaimVolumes(existing, []corev1.Volume{pvcVol}) require.Len(t, result, 2) - // config preserved + require.Equal(t, "config", result[0].Name) require.NotNil(t, result[0].ConfigMap) - // data replaced by PVC + require.Equal(t, "data", result[1].Name) require.NotNil(t, result[1].PersistentVolumeClaim) }) @@ -3356,10 +3338,6 @@ func TestMergeVolumeClaimVolumes(t *testing.T) { }) } -// TestSandboxReconcile_ConditionsDoNotAccumulate verifies that reconciling a -// ready sandbox many times does not grow the conditions slice. A bug -// that appends instead of upserts the Ready condition will cause unbounded -// status growth. func TestSandboxReconcile_ConditionsDoNotAccumulate(t *testing.T) { sbName := "no-grow-sandbox" sbNs := "default" diff --git a/docs/api.md b/docs/api.md index d73f9da..df30a00 100644 --- a/docs/api.md +++ b/docs/api.md @@ -16,6 +16,7 @@ Package v1alpha1 contains API Schema definitions for the agents v1alpha1 API gro ### Resource Types - [Sandbox](#sandbox) +- [SandboxList](#sandboxlist) @@ -34,9 +35,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name must be unique within a namespace. Is required when creating resources, although
some resources may allow a client to request the generation of an appropriate name
automatically. Name is primarily intended for creation idempotence and configuration
definition.
Cannot be updated.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names | | Optional: \{\}
| -| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | Optional: \{\}
| -| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | Optional: \{\}
| +| `name` _string_ | name must be unique within a namespace. Is required when creating resources, although
some resources may allow a client to request the generation of an appropriate name
automatically. Name is primarily intended for creation idempotence and configuration
definition.
Cannot be updated.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names | | | +| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | | +| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | | #### Lifecycle @@ -52,8 +53,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
Optional: \{\}
| -| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
Optional: \{\}
| +| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
| +| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
| #### PersistentVolumeClaimTemplate @@ -70,8 +71,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#persistentvolumeclaimspec-v1-core)_ | spec is the PVC's spec | | Required: \{\}
| +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#persistentvolumeclaimspec-v1-core)_ | spec is the PVC's spec | | | #### PodMetadata @@ -88,8 +89,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | Optional: \{\}
| -| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | Optional: \{\}
| +| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | | +| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | | #### PodTemplate @@ -106,8 +107,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `spec` _[PodSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#podspec-v1-core)_ | spec is the Pod's spec | | Required: \{\}
| -| `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[PodSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podspec-v1-core)_ | spec is the Pod's spec | | | +| `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | #### Sandbox @@ -118,17 +119,38 @@ Sandbox is the Schema for the sandboxes API. - +_Appears in:_ +- [SandboxList](#sandboxlist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `agents.x-k8s.io/v1alpha1` | | | | `kind` _string_ | `Sandbox` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxSpec](#sandboxspec)_ | spec defines the desired state of Sandbox | | Required: \{\}
| -| `status` _[SandboxStatus](#sandboxstatus)_ | status defines the observed state of Sandbox | | Optional: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxSpec](#sandboxspec)_ | spec defines the desired state of Sandbox | | | +| `status` _[SandboxStatus](#sandboxstatus)_ | status defines the observed state of Sandbox | | | + + +#### SandboxList + + + +SandboxList contains a list of Sandbox. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1alpha1` | | | +| `kind` _string_ | `SandboxList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[Sandbox](#sandbox) array_ | | | | #### SandboxSpec @@ -144,12 +166,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod spec that will be used to create an agent sandbox. | | Required: \{\}
| -| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
Every claim in this list must have at least one matching access mode with a provisioner volume. | | Optional: \{\}
| -| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
Optional: \{\}
| -| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
Optional: \{\}
| -| `replicas` _integer_ | replicas is the number of desired replicas.
The only allowed values are 0 and 1.
Defaults to 1. | 1 | Maximum: 1
Minimum: 0
Optional: \{\}
| -| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for this Sandbox.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod spec that will be used to create an agent sandbox. | | | +| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
Every claim in this list must have at least one matching access mode with a provisioner volume. | | | +| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
| +| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
| +| `replicas` _integer_ | replicas is the number of desired replicas.
The only allowed values are 0 and 1.
Defaults to 1. | 1 | Maximum: 1
Minimum: 0
| +| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for this Sandbox.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | | #### SandboxStatus @@ -165,12 +187,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `serviceFQDN` _string_ | serviceFQDN that is valid for default cluster settings
The domain defaults to cluster.local but is configurable via the controller's --cluster-domain flag. | | Optional: \{\}
| -| `service` _string_ | service is a sandbox-example | | Optional: \{\}
| -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | conditions defines the status conditions array | | Optional: \{\}
| -| `replicas` _integer_ | replicas is the number of actual replicas. | | Minimum: 0
Optional: \{\}
| -| `selector` _string_ | selector is the label selector for pods. | | Optional: \{\}
| -| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | Optional: \{\}
| +| `serviceFQDN` _string_ | serviceFQDN that is valid for default cluster settings
The domain defaults to cluster.local but is configurable via the controller's --cluster-domain flag. | | | +| `service` _string_ | service is a sandbox-example | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#condition-v1-meta) array_ | conditions defines the status conditions array | | | +| `replicas` _integer_ | replicas is the number of actual replicas. | | Minimum: 0
| +| `selector` _string_ | selector is the label selector for pods. | | | +| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | | #### ShutdownPolicy @@ -202,6 +224,13 @@ Package v1beta1 contains API Schema definitions for the agents v1beta1 API group ### Resource Types - [Sandbox](#sandbox) +- [SandboxForkOptions](#sandboxforkoptions) +- [SandboxForkResult](#sandboxforkresult) +- [SandboxList](#sandboxlist) +- [SandboxPauseOptions](#sandboxpauseoptions) +- [SandboxResumeOptions](#sandboxresumeoptions) +- [SandboxSnapshotOptions](#sandboxsnapshotoptions) +- [SandboxSnapshotResult](#sandboxsnapshotresult) @@ -220,9 +249,27 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name must be unique within a namespace. Is required when creating resources, although
some resources may allow a client to request the generation of an appropriate name
automatically. Name is primarily intended for creation idempotence and configuration
definition.
Cannot be updated.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names | | Optional: \{\}
| -| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | Optional: \{\}
| -| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | Optional: \{\}
| +| `name` _string_ | name must be unique within a namespace. Is required when creating resources, although
some resources may allow a client to request the generation of an appropriate name
automatically. Name is primarily intended for creation idempotence and configuration
definition.
Cannot be updated.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names | | | +| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | | +| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | | + + +#### ForkedSandbox + + + +ForkedSandbox identifies one child of a fork. + + + +_Appears in:_ +- [SandboxForkResult](#sandboxforkresult) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `sandboxID` _string_ | sandboxID is the child's node-local claim id. | | | +| `nodeName` _string_ | nodeName is the node that owns the child. A fork is node-local, so every
child lands on the source's node. | | | +| `address` _string_ | address is the child's connection address, when the node published one. | | | #### Lifecycle @@ -238,8 +285,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
Optional: \{\}
| -| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
Optional: \{\}
| +| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
| +| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
| #### PersistentVolumeClaimTemplate @@ -258,8 +305,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#persistentvolumeclaimspec-v1-core)_ | spec is the PVC's spec | | Required: \{\}
| +| `metadata` _[EmbeddedObjectMetadata](#embeddedobjectmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PersistentVolumeClaimSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#persistentvolumeclaimspec-v1-core)_ | spec is the PVC's spec | | | #### PodMetadata @@ -276,8 +323,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | Optional: \{\}
| -| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | Optional: \{\}
| +| `labels` _object (keys:string, values:string)_ | labels defines the map of string keys and values that can be used to organize and categorize
(scope and select) objects. May match selectors of replication controllers
and services.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels | | | +| `annotations` _object (keys:string, values:string)_ | annotations is an unstructured key value map stored with a resource that may be
set by external tools to store and retrieve arbitrary metadata. They are not
queryable and should be preserved when modifying objects.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations | | | #### PodTemplate @@ -295,8 +342,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `spec` _[PodSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#podspec-v1-core)_ | spec is the Pod's spec | | Required: \{\}
| -| `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[PodSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podspec-v1-core)_ | spec is the Pod's spec | | | +| `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | #### Sandbox @@ -307,17 +354,18 @@ Sandbox is the Schema for the sandboxes API. - +_Appears in:_ +- [SandboxList](#sandboxlist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | | `kind` _string_ | `Sandbox` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxSpec](#sandboxspec)_ | spec defines the desired state of Sandbox | | Required: \{\}
| -| `status` _[SandboxStatus](#sandboxstatus)_ | status defines the observed state of Sandbox | | Optional: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxSpec](#sandboxspec)_ | spec defines the desired state of Sandbox | | | +| `status` _[SandboxStatus](#sandboxstatus)_ | status defines the observed state of Sandbox | | | #### SandboxBlueprint @@ -335,9 +383,71 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod that will be created in the sandbox.
Note: When provisioned via a SandboxTemplate (such as by a SandboxClaim or SandboxWarmPool),
if AutomountServiceAccountToken is not specified in the PodSpec, the controller defaults it
to false to ensure a secure-by-default environment. | | Required: \{\}
| -| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
When creating a sandbox, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | Optional: \{\}
| -| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for the Sandbox workload.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod that will be created in the sandbox.
Note: When provisioned via a SandboxTemplate (such as by a SandboxClaim or SandboxWarmPool),
if AutomountServiceAccountToken is not specified in the PodSpec, the controller defaults it
to false to ensure a secure-by-default environment. | | | +| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
When creating a sandbox, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | | +| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for the Sandbox workload.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | | + + +#### SandboxForkOptions + + + +SandboxForkOptions is the body of POST sandboxes/{name}/fork. The source is +checkpointed in place and keeps running; each child is a brand-new sandbox +with its own id and lease, not a replica of the source's identity. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxForkOptions` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `count` _integer_ | count is how many children to branch. Defaults to 1, and is bounded by
the owning node's configured fork limit. | | | +| `ttlSeconds` _integer_ | ttlSeconds is each child's lease. Children never inherit the parent's
remaining lease — a lease is a per-sandbox resource bound. Zero takes the
node's default. | | | + + +#### SandboxForkResult + + + +SandboxForkResult is the reply to a fork: one entry per child, in request +order. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxForkResult` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `children` _[ForkedSandbox](#forkedsandbox) array_ | children are the branched sandboxes. | | | + + +#### SandboxList + + + +SandboxList contains a list of Sandbox. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[Sandbox](#sandbox) array_ | | | | #### SandboxOperatingMode @@ -357,6 +467,89 @@ _Appears in:_ | `Suspended` | SandboxOperatingModeSuspended indicates the sandbox should be suspended.
| +#### SandboxPauseOptions + + + +SandboxPauseOptions is the body of POST sandboxes/{name}/pause. Pausing +snapshots the guest's memory and stops its VM, so it costs time proportional +to that memory — unlike resume, which takes the mmap restore fast path. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxPauseOptions` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | + + +#### SandboxResumeOptions + + + +SandboxResumeOptions is the body of POST sandboxes/{name}/resume. Resuming a +paused sandbox restores it through cocoon's mmap fast path and is idempotent +on one that is already running. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxResumeOptions` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | + + +#### SandboxSnapshotOptions + + + +SandboxSnapshotOptions is the body of POST sandboxes/{name}/snapshot. The +source keeps running; the checkpoint is an immutable state later sandboxes +can branch from. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxSnapshotOptions` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `name` _string_ | name labels the checkpoint. Optional; the node assigns an id regardless. | | | + + +#### SandboxSnapshotResult + + + +SandboxSnapshotResult is the reply to a snapshot. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxSnapshotResult` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `snapshotID` _string_ | snapshotID is the checkpoint's node-local id. | | | +| `name` _string_ | name echoes the requested label, when one was given. | | | +| `nodeName` _string_ | nodeName is the node holding the checkpoint. Checkpoints are node-local,
so branching from or deleting one requires knowing its node. | | | +| `creationTimestamp` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | creationTimestamp is when the node captured the checkpoint. | | | + + #### SandboxSpec @@ -370,12 +563,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod that will be created in the sandbox.
Note: When provisioned via a SandboxTemplate (such as by a SandboxClaim or SandboxWarmPool),
if AutomountServiceAccountToken is not specified in the PodSpec, the controller defaults it
to false to ensure a secure-by-default environment. | | Required: \{\}
| -| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
When creating a sandbox, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | Optional: \{\}
| -| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for the Sandbox workload.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | Optional: \{\}
| -| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
Optional: \{\}
| -| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
Optional: \{\}
| -| `operatingMode` _[SandboxOperatingMode](#sandboxoperatingmode)_ | operatingMode specifies the desired operational state of the Sandbox.
Defaults to Running if not specified. | Running | Enum: [Running Suspended]
Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod that will be created in the sandbox.
Note: When provisioned via a SandboxTemplate (such as by a SandboxClaim or SandboxWarmPool),
if AutomountServiceAccountToken is not specified in the PodSpec, the controller defaults it
to false to ensure a secure-by-default environment. | | | +| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
When creating a sandbox, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | | +| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for the Sandbox workload.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | | +| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | shutdownTime is the absolute time when the sandbox expires. | | Format: date-time
| +| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines if the Sandbox resource itself should be deleted when it expires.
Underlying resources(Pods, Services) are always deleted on expiry. | Retain | Enum: [Delete Retain]
| +| `operatingMode` _[SandboxOperatingMode](#sandboxoperatingmode)_ | operatingMode specifies the desired operational state of the Sandbox.
Defaults to Running if not specified. | Running | Enum: [Running Suspended]
| #### SandboxStatus @@ -391,12 +584,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `serviceFQDN` _string_ | serviceFQDN that is valid for default cluster settings
The domain defaults to cluster.local but is configurable via the controller's --cluster-domain flag. | | Optional: \{\}
| -| `service` _string_ | service is a sandbox-example | | Optional: \{\}
| -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | conditions defines the status conditions array | | Optional: \{\}
| -| `selector` _string_ | selector is the label selector for pods. | | Optional: \{\}
| -| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | Optional: \{\}
| -| `nodeName` _string_ | nodeName is the name of the node where the underlying pod is scheduled. | | Optional: \{\}
| +| `serviceFQDN` _string_ | serviceFQDN that is valid for default cluster settings
The domain defaults to cluster.local but is configurable via the controller's --cluster-domain flag. | | | +| `service` _string_ | service is a sandbox-example | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#condition-v1-meta) array_ | conditions defines the status conditions array | | | +| `selector` _string_ | selector is the label selector for pods. | | | +| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | | +| `nodeName` _string_ | nodeName is the name of the node where the underlying pod is scheduled. | | | #### ShutdownPolicy @@ -427,8 +620,11 @@ Package v1alpha1 contains API Schema definitions for the extensions.agents v1alp ### Resource Types - [SandboxClaim](#sandboxclaim) +- [SandboxClaimList](#sandboxclaimlist) - [SandboxTemplate](#sandboxtemplate) +- [SandboxTemplateList](#sandboxtemplatelist) - [SandboxWarmPool](#sandboxwarmpool) +- [SandboxWarmPoolList](#sandboxwarmpoollist) @@ -445,9 +641,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name of the environment variable. | | Required: \{\}
| -| `value` _string_ | value of the environment variable. | | Required: \{\}
| -| `containerName` _string_ | containerName specifies the target container for the environment variable.
If not specified, it defaults to the first container defined in the template. | | Optional: \{\}
| +| `name` _string_ | name of the environment variable. | | | +| `value` _string_ | value of the environment variable. | | | +| `containerName` _string_ | containerName specifies the target container for the environment variable.
If not specified, it defaults to the first container defined in the template. | | | #### EnvVarsInjectionPolicy @@ -481,9 +677,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | shutdownTime is the absolute time when the SandboxClaim expires.
This time governs the lifecycle of the claim. It is not propagated to the
underlying Sandbox. Instead, the SandboxClaim controller enforces this
expiration by deleting the Sandbox resources when the time is reached.
If this field is omitted or set to nil, the SandboxClaim itself won't expire.
This implies unsetting a Sandbox's ShutdownTime via SandboxClaim isn't supported. | | Format: date-time
Optional: \{\}
| -| `ttlSecondsAfterFinished` _integer_ | ttlSecondsAfterFinished limits how long a finished claim is retained.
The timer starts from the mirrored Finished condition's LastTransitionTime. | | Minimum: 0
Optional: \{\}
| -| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines the behavior when the SandboxClaim expires. | Retain | Enum: [Delete DeleteForeground Retain]
Optional: \{\}
| +| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | shutdownTime is the absolute time when the SandboxClaim expires.
This time governs the lifecycle of the claim. It is not propagated to the
underlying Sandbox. Instead, the SandboxClaim controller enforces this
expiration by deleting the Sandbox resources when the time is reached.
If this field is omitted or set to nil, the SandboxClaim itself won't expire.
This implies unsetting a Sandbox's ShutdownTime via SandboxClaim isn't supported. | | Format: date-time
| +| `ttlSecondsAfterFinished` _integer_ | ttlSecondsAfterFinished limits how long a finished claim is retained.
The timer starts from the mirrored Finished condition's LastTransitionTime. | | Minimum: 0
| +| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines the behavior when the SandboxClaim expires. | Retain | Enum: [Delete DeleteForeground Retain]
| #### NetworkPolicyManagement @@ -517,8 +713,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `ingress` _[NetworkPolicyIngressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#networkpolicyingressrule-v1-networking) array_ | ingress is a list of ingress rules to be applied to the sandbox.
Traffic is allowed to the sandbox if it matches at least one rule.
If this list is empty, all ingress traffic is blocked (Default Deny). | | Optional: \{\}
| -| `egress` _[NetworkPolicyEgressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#networkpolicyegressrule-v1-networking) array_ | egress is a list of egress rules to be applied to the sandbox.
Traffic is allowed out of the sandbox if it matches at least one rule.
If this list is empty, all egress traffic is blocked (Default Deny). | | Optional: \{\}
| +| `ingress` _[NetworkPolicyIngressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#networkpolicyingressrule-v1-networking) array_ | ingress is a list of ingress rules to be applied to the sandbox.
Traffic is allowed to the sandbox if it matches at least one rule.
If this list is empty, all ingress traffic is blocked (Default Deny). | | | +| `egress` _[NetworkPolicyEgressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#networkpolicyegressrule-v1-networking) array_ | egress is a list of egress rules to be applied to the sandbox.
Traffic is allowed out of the sandbox if it matches at least one rule.
If this list is empty, all egress traffic is blocked (Default Deny). | | | #### SandboxClaim @@ -529,17 +725,38 @@ SandboxClaim is the Schema for the sandbox Claim API. - +_Appears in:_ +- [SandboxClaimList](#sandboxclaimlist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1alpha1` | | | | `kind` _string_ | `SandboxClaim` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxClaimSpec](#sandboxclaimspec)_ | spec defines the desired state of Sandbox | | Required: \{\}
| -| `status` _[SandboxClaimStatus](#sandboxclaimstatus)_ | status defines the observed state of Sandbox | | Optional: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxClaimSpec](#sandboxclaimspec)_ | spec defines the desired state of Sandbox | | | +| `status` _[SandboxClaimStatus](#sandboxclaimstatus)_ | status defines the observed state of Sandbox | | | + + +#### SandboxClaimList + + + +SandboxClaimList contains a list of SandboxClaim. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1alpha1` | | | +| `kind` _string_ | `SandboxClaimList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[SandboxClaim](#sandboxclaim) array_ | | | | #### SandboxClaimSpec @@ -555,11 +772,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `sandboxTemplateRef` _[SandboxTemplateRef](#sandboxtemplateref)_ | sandboxTemplateRef defines the name of the SandboxTemplate to be used for creating a Sandbox. | | Required: \{\}
| -| `lifecycle` _[Lifecycle](#lifecycle)_ | lifecycle defines when and how the SandboxClaim should be shut down. | | Optional: \{\}
| -| `warmpool` _[WarmPoolPolicy](#warmpoolpolicy)_ | warmpool specifies the warm pool policy for sandbox adoption.
- "none": Do not use any warm pool, always create fresh sandboxes
- "default": Use default behavior, select from all matching warm pools (default)
- A warm pool name: Select only from the specified warm pool (e.g., "fast-pool", "secure-pool") | default | Optional: \{\}
| -| `additionalPodMetadata` _[PodMetadata](#podmetadata)_ | additionalPodMetadata defines the labels and annotations to be propagated to the Sandbox Pod.
Label values are limited to 63 characters and must match Kubernetes label value patterns. | | Optional: \{\}
| -| `env` _[EnvVar](#envvar) array_ | env is a list of environment variables to inject into the sandbox | | Optional: \{\}
| +| `sandboxTemplateRef` _[SandboxTemplateRef](#sandboxtemplateref)_ | sandboxTemplateRef defines the name of the SandboxTemplate to be used for creating a Sandbox. | | | +| `lifecycle` _[Lifecycle](#lifecycle)_ | lifecycle defines when and how the SandboxClaim should be shut down. | | | +| `warmpool` _[WarmPoolPolicy](#warmpoolpolicy)_ | warmpool specifies the warm pool policy for sandbox adoption.
- "none": Do not use any warm pool, always create fresh sandboxes
- "default": Use default behavior, select from all matching warm pools (default)
- A warm pool name: Select only from the specified warm pool (e.g., "fast-pool", "secure-pool") | default | | +| `additionalPodMetadata` _[PodMetadata](#podmetadata)_ | additionalPodMetadata defines the labels and annotations to be propagated to the Sandbox Pod.
Label values are limited to 63 characters and must match Kubernetes label value patterns. | | | +| `env` _[EnvVar](#envvar) array_ | env is a list of environment variables to inject into the sandbox | | | #### SandboxClaimStatus @@ -575,8 +792,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | conditions represent the latest available observations of a Sandbox's current state. | | Optional: \{\}
| -| `sandbox` _[SandboxStatus](#sandboxstatus)_ | sandbox defines the state of Sandbox | | Optional: \{\}
| +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#condition-v1-meta) array_ | conditions represent the latest available observations of a Sandbox's current state. | | | +| `sandbox` _[SandboxStatus](#sandboxstatus)_ | sandbox defines the state of Sandbox | | | #### SandboxStatus @@ -592,8 +809,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name is the name of the Sandbox created from this claim | | Optional: \{\}
| -| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | Optional: \{\}
| +| `name` _string_ | name is the name of the Sandbox created from this claim | | | +| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | | #### SandboxTemplate @@ -604,16 +821,37 @@ SandboxTemplate is the Schema for the sandbox template API. - +_Appears in:_ +- [SandboxTemplateList](#sandboxtemplatelist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1alpha1` | | | | `kind` _string_ | `SandboxTemplate` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxTemplateSpec](#sandboxtemplatespec)_ | spec defines the desired state of Sandbox | | Required: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxTemplateSpec](#sandboxtemplatespec)_ | spec defines the desired state of Sandbox | | | + + +#### SandboxTemplateList + + + +SandboxTemplateList contains a list of Sandbox. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1alpha1` | | | +| `kind` _string_ | `SandboxTemplateList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[SandboxTemplate](#sandboxtemplate) array_ | | | | #### SandboxTemplateRef @@ -630,7 +868,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name of the SandboxTemplate | | Required: \{\}
| +| `name` _string_ | name of the SandboxTemplate | | | #### SandboxTemplateSpec @@ -646,12 +884,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate defines the object template that describes the pod spec that will be used to create
an agent sandbox.
If AutomountServiceAccountToken is not specified in the PodSpec, it defaults to false
to ensure a secure-by-default environment. | | Required: \{\}
| -| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that pods created from this template
are allowed to reference. When a SandboxClaim or SandboxWarmPool creates a sandbox
from this template, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | Optional: \{\}
| -| `networkPolicy` _[NetworkPolicySpec](#networkpolicyspec)_ | networkPolicy defines the network policy to be applied to the sandboxes
created from this template. A single shared NetworkPolicy is created per Template.
Behavior is dictated by the NetworkPolicyManagement field:
- If Management is "Unmanaged": This field is completely ignored.
- If Management is "Managed" (default) and this field is omitted (nil): The controller
automatically applies a strict Secure Default policy:
* Ingress: Allow traffic only from the Sandbox Router.
* Egress: Allow Public Internet only. Blocks internal IPs (RFC1918), Metadata Server, etc.
- If Management is "Managed" and this field is provided: The controller applies your custom rules.
Update Behavior:
Because the NetworkPolicy is shared at the template level, any updates to these rules
will be applied to the single shared policy object. The underlying Kubernetes CNI will then
dynamically enforce the updated rules across all existing and future sandboxes
referencing this template.
NOTE: This is a restricted subset of the standard Kubernetes NetworkPolicySpec.
Fields like 'PodSelector' and 'PolicyTypes' are intentionally excluded because
they are managed by the controller to ensure strict isolation and default-deny posture.
WARNING: This policy enforces a strict "Default Deny" ingress posture.
If your Pod uses sidecars (e.g., Istio proxy, monitoring agents) that listen
on their own ports, the NetworkPolicy will BLOCK traffic to them by default.
You MUST explicitly allow traffic to these sidecar ports using 'Ingress',
otherwise the sidecars may fail health checks. | | Optional: \{\}
| -| `networkPolicyManagement` _[NetworkPolicyManagement](#networkpolicymanagement)_ | networkPolicyManagement defines whether the controller manages the NetworkPolicy.
Valid values are "Managed" (default) or "Unmanaged". | Managed | Enum: [Managed Unmanaged]
Optional: \{\}
| -| `envVarsInjectionPolicy` _[EnvVarsInjectionPolicy](#envvarsinjectionpolicy)_ | envVarsInjectionPolicy allows a SandboxClaim to inject or override environment variables defined in the template.
If set to Disallowed, the SandboxClaim will be rejected if it specifies any environment variables. | Disallowed | Enum: [Allowed Overrides Disallowed]
Optional: \{\}
| -| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for Sandboxes created from this template.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate defines the object template that describes the pod spec that will be used to create
an agent sandbox.
If AutomountServiceAccountToken is not specified in the PodSpec, it defaults to false
to ensure a secure-by-default environment. | | | +| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that pods created from this template
are allowed to reference. When a SandboxClaim or SandboxWarmPool creates a sandbox
from this template, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | | +| `networkPolicy` _[NetworkPolicySpec](#networkpolicyspec)_ | networkPolicy defines the network policy to be applied to the sandboxes
created from this template. A single shared NetworkPolicy is created per Template.
Behavior is dictated by the NetworkPolicyManagement field:
- If Management is "Unmanaged": This field is completely ignored.
- If Management is "Managed" (default) and this field is omitted (nil): The controller
automatically applies a strict Secure Default policy:
* Ingress: Allow traffic only from the Sandbox Router.
* Egress: Allow Public Internet only. Blocks internal IPs (RFC1918), Metadata Server, etc.
- If Management is "Managed" and this field is provided: The controller applies your custom rules.
Update Behavior:
Because the NetworkPolicy is shared at the template level, any updates to these rules
will be applied to the single shared policy object. The underlying Kubernetes CNI will then
dynamically enforce the updated rules across all existing and future sandboxes
referencing this template.
NOTE: This is a restricted subset of the standard Kubernetes NetworkPolicySpec.
Fields like 'PodSelector' and 'PolicyTypes' are intentionally excluded because
they are managed by the controller to ensure strict isolation and default-deny posture.
WARNING: This policy enforces a strict "Default Deny" ingress posture.
If your Pod uses sidecars (e.g., Istio proxy, monitoring agents) that listen
on their own ports, the NetworkPolicy will BLOCK traffic to them by default.
You MUST explicitly allow traffic to these sidecar ports using 'Ingress',
otherwise the sidecars may fail health checks. | | | +| `networkPolicyManagement` _[NetworkPolicyManagement](#networkpolicymanagement)_ | networkPolicyManagement defines whether the controller manages the NetworkPolicy.
Valid values are "Managed" (default) or "Unmanaged". | Managed | Enum: [Managed Unmanaged]
| +| `envVarsInjectionPolicy` _[EnvVarsInjectionPolicy](#envvarsinjectionpolicy)_ | envVarsInjectionPolicy allows a SandboxClaim to inject or override environment variables defined in the template.
If set to Disallowed, the SandboxClaim will be rejected if it specifies any environment variables. | Disallowed | Enum: [Allowed Overrides Disallowed]
| +| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for Sandboxes created from this template.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | | #### SandboxWarmPool @@ -662,17 +900,38 @@ SandboxWarmPool is the Schema for the sandboxwarmpools API. - +_Appears in:_ +- [SandboxWarmPoolList](#sandboxwarmpoollist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1alpha1` | | | | `kind` _string_ | `SandboxWarmPool` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxWarmPoolSpec](#sandboxwarmpoolspec)_ | spec defines the desired state of SandboxWarmPool | | Required: \{\}
| -| `status` _[SandboxWarmPoolStatus](#sandboxwarmpoolstatus)_ | status defines the observed state of SandboxWarmPool | | Optional: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxWarmPoolSpec](#sandboxwarmpoolspec)_ | spec defines the desired state of SandboxWarmPool | | | +| `status` _[SandboxWarmPoolStatus](#sandboxwarmpoolstatus)_ | status defines the observed state of SandboxWarmPool | | | + + +#### SandboxWarmPoolList + + + +SandboxWarmPoolList contains a list of SandboxWarmPool. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1alpha1` | | | +| `kind` _string_ | `SandboxWarmPoolList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[SandboxWarmPool](#sandboxwarmpool) array_ | | | | #### SandboxWarmPoolSpec @@ -688,9 +947,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `replicas` _integer_ | replicas is the desired number of sandboxes in the pool.
This field is controlled by an HPA if specified. | | Minimum: 0
Required: \{\}
| -| `sandboxTemplateRef` _[SandboxTemplateRef](#sandboxtemplateref)_ | sandboxTemplateRef - name of the SandboxTemplate to be used for creating a Sandbox
Warning: Any change to the json tag "sandboxTemplateRef" must be synchronized with the TemplateRefField constant. | | Required: \{\}
| -| `updateStrategy` _[SandboxWarmPoolUpdateStrategy](#sandboxwarmpoolupdatestrategy)_ | updateStrategy - strategy for updating the SandboxWarmPool pods based on sandboxTemplateRef name change or underlying template changes | | Optional: \{\}
| +| `replicas` _integer_ | replicas is the desired number of sandboxes in the pool.
This field is controlled by an HPA if specified. | | Minimum: 0
| +| `sandboxTemplateRef` _[SandboxTemplateRef](#sandboxtemplateref)_ | sandboxTemplateRef - name of the SandboxTemplate to be used for creating a Sandbox
Warning: Any change to the json tag "sandboxTemplateRef" must be synchronized with the TemplateRefField constant. | | | +| `updateStrategy` _[SandboxWarmPoolUpdateStrategy](#sandboxwarmpoolupdatestrategy)_ | updateStrategy - strategy for updating the SandboxWarmPool pods based on sandboxTemplateRef name change or underlying template changes | | | #### SandboxWarmPoolStatus @@ -706,9 +965,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `replicas` _integer_ | replicas is the total number of sandboxes in the pool. | | Optional: \{\}
| -| `readyReplicas` _integer_ | readyReplicas is the total number of sandboxes in the pool that are in a ready state. | | Optional: \{\}
| -| `selector` _string_ | selector is the label selector used to find the pods in the pool. | | Optional: \{\}
| +| `replicas` _integer_ | replicas is the total number of sandboxes in the pool. | | | +| `readyReplicas` _integer_ | readyReplicas is the total number of sandboxes in the pool that are in a ready state. | | | +| `selector` _string_ | selector is the label selector used to find the pods in the pool. | | | #### SandboxWarmPoolUpdateStrategy @@ -724,7 +983,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[SandboxWarmPoolUpdateStrategyType](#sandboxwarmpoolupdatestrategytype)_ | type indicates the type of the SandboxWarmPoolUpdateStrategy.
Default is OnReplenish. | OnReplenish | Enum: [Recreate OnReplenish]
Optional: \{\}
| +| `type` _[SandboxWarmPoolUpdateStrategyType](#sandboxwarmpoolupdatestrategytype)_ | type indicates the type of the SandboxWarmPoolUpdateStrategy.
Default is OnReplenish. | OnReplenish | Enum: [Recreate OnReplenish]
| #### SandboxWarmPoolUpdateStrategyType @@ -794,9 +1053,14 @@ Package v1beta1 contains API Schema definitions for the extensions v1beta1 API g Package v1beta1 contains API Schema definitions for the extensions.agents v1beta1 API group. ### Resource Types +- [NodeInventory](#nodeinventory) +- [NodeInventoryList](#nodeinventorylist) - [SandboxClaim](#sandboxclaim) +- [SandboxClaimList](#sandboxclaimlist) - [SandboxTemplate](#sandboxtemplate) +- [SandboxTemplateList](#sandboxtemplatelist) - [SandboxWarmPool](#sandboxwarmpool) +- [SandboxWarmPoolList](#sandboxwarmpoollist) @@ -813,9 +1077,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name of the environment variable. | | Required: \{\}
| -| `value` _string_ | value of the environment variable. | | Required: \{\}
| -| `containerName` _string_ | containerName specifies the target container for the environment variable.
If not specified, it defaults to the first container defined in the template. | | Optional: \{\}
| +| `name` _string_ | name of the environment variable. | | | +| `value` _string_ | value of the environment variable. | | | +| `containerName` _string_ | containerName specifies the target container for the environment variable.
If not specified, it defaults to the first container defined in the template. | | | #### EnvVarsInjectionPolicy @@ -836,6 +1100,28 @@ _Appears in:_ | `Disallowed` | EnvVarsInjectionPolicyDisallowed prevents a SandboxClaim from injecting any environment variables.
| +#### InventoryEntry + + + +InventoryEntry is one live sandbox as summarized by its owning node. + + + +_Appears in:_ +- [NodeInventory](#nodeinventory) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | name is the sandbox "/"; an unqualified name means the
default namespace. | | | +| `id` _string_ | id is the owning node's sandboxd claim id ("sb_..."), the handle its
sandbox-release verb needs. The aggregated apiserver surfaces it on the
synthesized Sandbox so Delete can release exactly this node-local microVM
(releasing by k8s name would target the wrong claim). Empty until the
node publishes it. | | | +| `phase` _string_ | phase is the node-reported sandbox phase (e.g. Running). | | | +| `template` _string_ | template is the pool template (base image) the sandbox was claimed from.
It is the only place the aggregated read path can recover it: no
per-sandbox object holds the pod spec. | | | +| `claimRef` _string_ | claimRef is the "/" of the SandboxClaim the sandbox is
bound to, if any. | | | +| `addr` _string_ | addr is the sandbox "host:port" address, if published. | | | +| `deadline` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | deadline is the node-granted lease expiry, if published. | | | + + #### Lifecycle @@ -849,9 +1135,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | shutdownTime is the absolute time when the SandboxClaim expires.
This time governs the lifecycle of the claim. It is not propagated to the
underlying Sandbox. Instead, the SandboxClaim controller enforces this
expiration by deleting the Sandbox resources when the time is reached.
If this field is omitted or set to nil, the SandboxClaim itself won't expire.
This implies unsetting a Sandbox's ShutdownTime via SandboxClaim isn't supported. | | Format: date-time
Optional: \{\}
| -| `ttlSecondsAfterFinished` _integer_ | ttlSecondsAfterFinished limits how long a finished claim is retained.
The timer starts from the mirrored Finished condition's LastTransitionTime. | | Minimum: 0
Optional: \{\}
| -| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines the behavior when the SandboxClaim expires. | Retain | Enum: [Delete DeleteForeground Retain]
Optional: \{\}
| +| `shutdownTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#time-v1-meta)_ | shutdownTime is the absolute time when the SandboxClaim expires.
This time governs the lifecycle of the claim. It is not propagated to the
underlying Sandbox. Instead, the SandboxClaim controller enforces this
expiration by deleting the Sandbox resources when the time is reached.
If this field is omitted or set to nil, the SandboxClaim itself won't expire.
This implies unsetting a Sandbox's ShutdownTime via SandboxClaim isn't supported. | | Format: date-time
| +| `ttlSecondsAfterFinished` _integer_ | ttlSecondsAfterFinished limits how long a finished claim is retained.
The timer starts from the mirrored Finished condition's LastTransitionTime. | | Minimum: 0
| +| `shutdownPolicy` _[ShutdownPolicy](#shutdownpolicy)_ | shutdownPolicy determines the behavior when the SandboxClaim expires. | Retain | Enum: [Delete DeleteForeground Retain]
| #### NetworkPolicyManagement @@ -885,8 +1171,81 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `ingress` _[NetworkPolicyIngressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#networkpolicyingressrule-v1-networking) array_ | ingress is a list of ingress rules to be applied to the sandbox.
Traffic is allowed to the sandbox if it matches at least one rule.
If this list is empty, all ingress traffic is blocked (Default Deny). | | Optional: \{\}
| -| `egress` _[NetworkPolicyEgressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#networkpolicyegressrule-v1-networking) array_ | egress is a list of egress rules to be applied to the sandbox.
Traffic is allowed out of the sandbox if it matches at least one rule.
If this list is empty, all egress traffic is blocked (Default Deny). | | Optional: \{\}
| +| `ingress` _[NetworkPolicyIngressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#networkpolicyingressrule-v1-networking) array_ | ingress is a list of ingress rules to be applied to the sandbox.
Traffic is allowed to the sandbox if it matches at least one rule.
If this list is empty, all ingress traffic is blocked (Default Deny). | | | +| `egress` _[NetworkPolicyEgressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#networkpolicyegressrule-v1-networking) array_ | egress is a list of egress rules to be applied to the sandbox.
Traffic is allowed out of the sandbox if it matches at least one rule.
If this list is empty, all egress traffic is blocked (Default Deny). | | | + + +#### NodeInventory + + + +NodeInventory is the single O(nodes) etcd object per node: the durable summary +of that node's live sandboxes, server-side-applied on a slow cadence and +scatter-gathered by the aggregated sandbox-apiserver. It is deliberately +spec-less (pure reported summary, no desired state) and cluster-scoped with +metadata.name equal to the node name. It lives in this CRD extensions group — +NOT in the aggregated agents.x-k8s.io group, whose entire v1beta1 the +APIService hands to the aggregated server (which serves only `sandboxes`). + + + +_Appears in:_ +- [NodeInventoryList](#nodeinventorylist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `NodeInventory` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `node` _string_ | node is the owning node name; it matches metadata.name. | | | +| `entries` _[InventoryEntry](#inventoryentry) array_ | entries summarizes the node's live sandboxes. | | | +| `address` _string_ | address is the node's sandboxd advertise address ("host:port"); the
aggregated apiserver routes a claim to this node's sandboxd through it. | | | +| `pools` _[PoolCapacity](#poolcapacity) array_ | pools is the node's per-pool warm capacity, used to pick a node that
already holds a warm microVM for a requested (template, net, size). | | | + + +#### NodeInventoryList + + + +NodeInventoryList contains a list of NodeInventory. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `NodeInventoryList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[NodeInventory](#nodeinventory) array_ | | | | + + +#### PoolCapacity + + + +PoolCapacity is one sandboxd warm pool's capacity as reported by its owning +node's GET /v1/info: the pool key plus its warm/target counts. The aggregated +apiserver reads it to pick a node that already holds a warm microVM for a +requested (template, net, size). + + + +_Appears in:_ +- [NodeInventory](#nodeinventory) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `template` _string_ | template is the pool's base image (the sandbox template). | | | +| `net` _string_ | net is the pool's network shape (e.g. "none", "egress"). | | | +| `size` _string_ | size is the pool's VM size class (e.g. "small"). | | | +| `warm` _integer_ | warm is the number of ready-to-claim warm microVMs currently in the pool. | | | +| `target` _integer_ | target is the pool's desired warm depth. | | | #### SandboxClaim @@ -897,17 +1256,38 @@ SandboxClaim is the Schema for the sandbox Claim API. - +_Appears in:_ +- [SandboxClaimList](#sandboxclaimlist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | | `kind` _string_ | `SandboxClaim` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxClaimSpec](#sandboxclaimspec)_ | spec defines the desired state of Sandbox | | Required: \{\}
| -| `status` _[SandboxClaimStatus](#sandboxclaimstatus)_ | status defines the observed state of Sandbox | | Optional: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxClaimSpec](#sandboxclaimspec)_ | spec defines the desired state of Sandbox | | | +| `status` _[SandboxClaimStatus](#sandboxclaimstatus)_ | status defines the observed state of Sandbox | | | + + +#### SandboxClaimList + + + +SandboxList contains a list of Sandbox. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxClaimList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[SandboxClaim](#sandboxclaim) array_ | | | | #### SandboxClaimSpec @@ -923,11 +1303,11 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `warmPoolRef` _[SandboxWarmPoolRef](#sandboxwarmpoolref)_ | warmPoolRef targets the specific pre-warmed infrastructure pool to check out from. | | Required: \{\}
| -| `lifecycle` _[Lifecycle](#lifecycle)_ | lifecycle defines when and how the SandboxClaim should be shut down. | | Optional: \{\}
| -| `additionalPodMetadata` _[PodMetadata](#podmetadata)_ | additionalPodMetadata defines the labels and annotations to be propagated to the Sandbox Pod.
Label values are limited to 63 characters and must match Kubernetes label value patterns. | | Optional: \{\}
| -| `env` _[EnvVar](#envvar) array_ | env is a list of environment variables to inject into the sandbox.
Please note adding this field means the Sandbox will always be cold-started from the
template of the warmpool. | | Optional: \{\}
| -| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of persistent volume claims to be created for the sandbox.
Specifying this field forces a cold start because warm pool pods will not have these volumes. | | Optional: \{\}
| +| `warmPoolRef` _[SandboxWarmPoolRef](#sandboxwarmpoolref)_ | warmPoolRef targets the specific pre-warmed infrastructure pool to check out from. | | | +| `lifecycle` _[Lifecycle](#lifecycle)_ | lifecycle defines when and how the SandboxClaim should be shut down. | | | +| `additionalPodMetadata` _[PodMetadata](#podmetadata)_ | additionalPodMetadata defines the labels and annotations to be propagated to the Sandbox Pod.
Label values are limited to 63 characters and must match Kubernetes label value patterns. | | | +| `env` _[EnvVar](#envvar) array_ | env is a list of environment variables to inject into the sandbox.
Please note adding this field means the Sandbox will always be cold-started from the
template of the warmpool. | | | +| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of persistent volume claims to be created for the sandbox.
Specifying this field forces a cold start because warm pool pods will not have these volumes. | | | #### SandboxClaimStatus @@ -943,8 +1323,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | conditions represent the latest available observations of a Sandbox's current state. | | Optional: \{\}
| -| `sandbox` _[SandboxStatus](#sandboxstatus)_ | sandbox defines the state of Sandbox | | Optional: \{\}
| +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#condition-v1-meta) array_ | conditions represent the latest available observations of a Sandbox's current state. | | | +| `sandbox` _[SandboxStatus](#sandboxstatus)_ | sandbox defines the state of Sandbox | | | #### SandboxStatus @@ -960,8 +1340,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name is the name of the Sandbox created from this claim | | Optional: \{\}
| -| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | Optional: \{\}
| +| `name` _string_ | name is the name of the Sandbox created from this claim | | | +| `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.
A pod may have multiple IPs in dual-stack clusters. | | | #### SandboxTemplate @@ -972,16 +1352,37 @@ SandboxTemplate is the Schema for the sandbox template API. - +_Appears in:_ +- [SandboxTemplateList](#sandboxtemplatelist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | | `kind` _string_ | `SandboxTemplate` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxTemplateSpec](#sandboxtemplatespec)_ | spec defines the desired state of Sandbox | | Required: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxTemplateSpec](#sandboxtemplatespec)_ | spec defines the desired state of Sandbox | | | + + +#### SandboxTemplateList + + + +SandboxTemplateList contains a list of Sandbox. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxTemplateList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[SandboxTemplate](#sandboxtemplate) array_ | | | | #### SandboxTemplateRef @@ -997,7 +1398,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name of the SandboxTemplate | | Required: \{\}
| +| `name` _string_ | name of the SandboxTemplate | | | #### SandboxTemplateSpec @@ -1013,13 +1414,13 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod that will be created in the sandbox.
Note: When provisioned via a SandboxTemplate (such as by a SandboxClaim or SandboxWarmPool),
if AutomountServiceAccountToken is not specified in the PodSpec, the controller defaults it
to false to ensure a secure-by-default environment. | | Required: \{\}
| -| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
When creating a sandbox, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | Optional: \{\}
| -| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for the Sandbox workload.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | Optional: \{\}
| -| `networkPolicy` _[NetworkPolicySpec](#networkpolicyspec)_ | networkPolicy defines the network policy to be applied to the sandboxes
created from this template. A single shared NetworkPolicy is created per Template.
Behavior is dictated by the NetworkPolicyManagement field:
- If Management is "Unmanaged": This field is completely ignored.
- If Management is "Managed" (default) and this field is omitted (nil): The controller
automatically applies a strict Secure Default policy:
* Ingress: Allow traffic only from the Sandbox Router.
* Egress: Allow Public Internet only. Blocks internal IPs (RFC1918), Metadata Server, etc.
- If Management is "Managed" and this field is provided: The controller applies your custom rules.
Update Behavior:
Because the NetworkPolicy is shared at the template level, any updates to these rules
will be applied to the single shared policy object. The underlying Kubernetes CNI will then
dynamically enforce the updated rules across all existing and future sandboxes
referencing this template.
NOTE: This is a restricted subset of the standard Kubernetes NetworkPolicySpec.
Fields like 'PodSelector' and 'PolicyTypes' are intentionally excluded because
they are managed by the controller to ensure strict isolation and default-deny posture.
WARNING: This policy enforces a strict "Default Deny" ingress posture.
If your Pod uses sidecars (e.g., Istio proxy, monitoring agents) that listen
on their own ports, the NetworkPolicy will BLOCK traffic to them by default.
You MUST explicitly allow traffic to these sidecar ports using 'Ingress',
otherwise the sidecars may fail health checks. | | Optional: \{\}
| -| `networkPolicyManagement` _[NetworkPolicyManagement](#networkpolicymanagement)_ | networkPolicyManagement defines whether the controller manages the NetworkPolicy.
Valid values are "Managed" (default) or "Unmanaged". | Managed | Enum: [Managed Unmanaged]
Optional: \{\}
| -| `envVarsInjectionPolicy` _[EnvVarsInjectionPolicy](#envvarsinjectionpolicy)_ | envVarsInjectionPolicy allows a SandboxClaim to inject or override environment variables defined in the template.
If set to Disallowed, the SandboxClaim will be rejected if it specifies any environment variables. | Disallowed | Enum: [Allowed Overrides Disallowed]
Optional: \{\}
| -| `volumeClaimTemplatesPolicy` _[VolumeClaimTemplatesPolicy](#volumeclaimtemplatespolicy)_ | volumeClaimTemplatesPolicy allows a SandboxClaim to inject or override volume claim templates defined in the template.
If set to Disallowed, the SandboxClaim will be rejected if it specifies any volume claim templates. | Disallowed | Enum: [Disallowed Allowed Overrides]
Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | podTemplate describes the pod that will be created in the sandbox.
Note: When provisioned via a SandboxTemplate (such as by a SandboxClaim or SandboxWarmPool),
if AutomountServiceAccountToken is not specified in the PodSpec, the controller defaults it
to false to ensure a secure-by-default environment. | | | +| `volumeClaimTemplates` _[PersistentVolumeClaimTemplate](#persistentvolumeclaimtemplate) array_ | volumeClaimTemplates is a list of claims that the sandbox pod is allowed to reference.
When creating a sandbox, PVCs will be created from these templates.
Every claim in this list must have at least one matching access mode with a provisioner volume.
NOTE: This list is atomic. Updates to this field will replace the entire list rather than merging with existing entries. | | | +| `service` _boolean_ | service controls whether the controller should automatically create a
headless Service for the Sandbox workload.
When unset, the controller preserves existing Services for backward
compatibility but does not create new ones. Set to true to enable or false
to explicitly disable and remove the Service. | | | +| `networkPolicy` _[NetworkPolicySpec](#networkpolicyspec)_ | networkPolicy defines the network policy to be applied to the sandboxes
created from this template. A single shared NetworkPolicy is created per Template.
Behavior is dictated by the NetworkPolicyManagement field:
- If Management is "Unmanaged": This field is completely ignored.
- If Management is "Managed" (default) and this field is omitted (nil): The controller
automatically applies a strict Secure Default policy:
* Ingress: Allow traffic only from the Sandbox Router.
* Egress: Allow Public Internet only. Blocks internal IPs (RFC1918), Metadata Server, etc.
- If Management is "Managed" and this field is provided: The controller applies your custom rules.
Update Behavior:
Because the NetworkPolicy is shared at the template level, any updates to these rules
will be applied to the single shared policy object. The underlying Kubernetes CNI will then
dynamically enforce the updated rules across all existing and future sandboxes
referencing this template.
NOTE: This is a restricted subset of the standard Kubernetes NetworkPolicySpec.
Fields like 'PodSelector' and 'PolicyTypes' are intentionally excluded because
they are managed by the controller to ensure strict isolation and default-deny posture.
WARNING: This policy enforces a strict "Default Deny" ingress posture.
If your Pod uses sidecars (e.g., Istio proxy, monitoring agents) that listen
on their own ports, the NetworkPolicy will BLOCK traffic to them by default.
You MUST explicitly allow traffic to these sidecar ports using 'Ingress',
otherwise the sidecars may fail health checks. | | | +| `networkPolicyManagement` _[NetworkPolicyManagement](#networkpolicymanagement)_ | networkPolicyManagement defines whether the controller manages the NetworkPolicy.
Valid values are "Managed" (default) or "Unmanaged". | Managed | Enum: [Managed Unmanaged]
| +| `envVarsInjectionPolicy` _[EnvVarsInjectionPolicy](#envvarsinjectionpolicy)_ | envVarsInjectionPolicy allows a SandboxClaim to inject or override environment variables defined in the template.
If set to Disallowed, the SandboxClaim will be rejected if it specifies any environment variables. | Disallowed | Enum: [Allowed Overrides Disallowed]
| +| `volumeClaimTemplatesPolicy` _[VolumeClaimTemplatesPolicy](#volumeclaimtemplatespolicy)_ | volumeClaimTemplatesPolicy allows a SandboxClaim to inject or override volume claim templates defined in the template.
If set to Disallowed, the SandboxClaim will be rejected if it specifies any volume claim templates. | Disallowed | Enum: [Disallowed Allowed Overrides]
| #### SandboxWarmPool @@ -1030,17 +1431,38 @@ SandboxWarmPool is the Schema for the sandboxwarmpools API. - +_Appears in:_ +- [SandboxWarmPoolList](#sandboxwarmpoollist) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | | `kind` _string_ | `SandboxWarmPool` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | Optional: \{\}
| -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | Optional: \{\}
| -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| -| `spec` _[SandboxWarmPoolSpec](#sandboxwarmpoolspec)_ | spec defines the desired state of SandboxWarmPool | | Required: \{\}
| -| `status` _[SandboxWarmPoolStatus](#sandboxwarmpoolstatus)_ | status defines the observed state of SandboxWarmPool | | Optional: \{\}
| +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[SandboxWarmPoolSpec](#sandboxwarmpoolspec)_ | spec defines the desired state of SandboxWarmPool | | | +| `status` _[SandboxWarmPoolStatus](#sandboxwarmpoolstatus)_ | status defines the observed state of SandboxWarmPool | | | + + +#### SandboxWarmPoolList + + + +SandboxWarmPoolList contains a list of SandboxWarmPool. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `extensions.agents.x-k8s.io/v1beta1` | | | +| `kind` _string_ | `SandboxWarmPoolList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[SandboxWarmPool](#sandboxwarmpool) array_ | | | | #### SandboxWarmPoolRef @@ -1056,7 +1478,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | name of the SandboxWarmPool | | Required: \{\}
| +| `name` _string_ | name of the SandboxWarmPool | | | #### SandboxWarmPoolSpec @@ -1072,9 +1494,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `replicas` _integer_ | replicas is the desired number of sandboxes in the pool.
This field is controlled by an HPA if specified. | 1 | Minimum: 0
Optional: \{\}
| -| `sandboxTemplateRef` _[SandboxTemplateRef](#sandboxtemplateref)_ | sandboxTemplateRef - name of the SandboxTemplate to be used for creating a Sandbox
Warning: Any change to the json tag "sandboxTemplateRef" must be synchronized with the TemplateRefField constant. | | Required: \{\}
| -| `updateStrategy` _[SandboxWarmPoolUpdateStrategy](#sandboxwarmpoolupdatestrategy)_ | updateStrategy - strategy for updating the SandboxWarmPool pods based on sandboxTemplateRef name change or underlying template changes | | Optional: \{\}
| +| `replicas` _integer_ | replicas is the desired number of sandboxes in the pool.
This field is controlled by an HPA if specified. | 1 | Minimum: 0
| +| `sandboxTemplateRef` _[SandboxTemplateRef](#sandboxtemplateref)_ | sandboxTemplateRef - name of the SandboxTemplate to be used for creating a Sandbox
Warning: Any change to the json tag "sandboxTemplateRef" must be synchronized with the TemplateRefField constant. | | | +| `updateStrategy` _[SandboxWarmPoolUpdateStrategy](#sandboxwarmpoolupdatestrategy)_ | updateStrategy - strategy for updating the SandboxWarmPool pods based on sandboxTemplateRef name change or underlying template changes | | | #### SandboxWarmPoolStatus @@ -1090,9 +1512,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `replicas` _integer_ | replicas is the total number of sandboxes in the pool. | | Optional: \{\}
| -| `readyReplicas` _integer_ | readyReplicas is the total number of sandboxes in the pool that are in a ready state. | | Optional: \{\}
| -| `selector` _string_ | selector is the label selector used to find the pods in the pool. | | Optional: \{\}
| +| `replicas` _integer_ | replicas is the total number of sandboxes in the pool. | | | +| `readyReplicas` _integer_ | readyReplicas is the total number of sandboxes in the pool that are in a ready state. | | | +| `selector` _string_ | selector is the label selector used to find the pods in the pool. | | | #### SandboxWarmPoolUpdateStrategy @@ -1108,7 +1530,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `type` _[SandboxWarmPoolUpdateStrategyType](#sandboxwarmpoolupdatestrategytype)_ | type indicates the type of the SandboxWarmPoolUpdateStrategy.
Default is OnReplenish. | OnReplenish | Enum: [Recreate OnReplenish]
Optional: \{\}
| +| `type` _[SandboxWarmPoolUpdateStrategyType](#sandboxwarmpoolupdatestrategytype)_ | type indicates the type of the SandboxWarmPoolUpdateStrategy.
Default is OnReplenish. | OnReplenish | Enum: [Recreate OnReplenish]
| #### SandboxWarmPoolUpdateStrategyType diff --git a/docs/configuration.md b/docs/configuration.md index 4991878..dbc0045 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,7 +7,8 @@ standard kubelet scheduling. ## Runtime and API surface - `--default-runtime` (`standard`): default backend for newly created Sandbox - Pods. Valid values are `vk-cocoon` and `standard`. + Pods. Valid values are `standard`, `vk-cocoon`, and `sandboxd`; `sandboxd` + routes Sandbox Pods to the vk-sandbox hot-pool virtual node. - `--extensions` (`true`): enable `SandboxTemplate`, `SandboxWarmPool`, and `SandboxClaim` controllers and webhooks. - `--cluster-domain` (`cluster.local`): suffix used to construct Sandbox @@ -17,6 +18,15 @@ An explicit Pod-template `runtimeClassName` always selects standard kubelet. An explicit `sandbox.cocoonstack.io/runtime` annotation takes precedence over the default. +## Cache scoping + +The operator's Pod, Service, and PersistentVolumeClaim informers are label +scoped to `agents.x-k8s.io/sandbox-name-hash`, so they watch only the children +the operator itself labels rather than every object in the cluster. A Pod, +Service, or PVC created outside the operator is therefore invisible to it: an +external warm pool that wants its objects adopted must set that label, and the +`agents.x-k8s.io/adoptable` label alone is not enough. + ## Controller concurrency The defaults are the configuration [PERFORMANCE.md](https://github.com/cocoonstack/sandbox-operator/blob/master/PERFORMANCE.md) @@ -28,6 +38,10 @@ was measured with, so an out-of-box install reproduces the published numbers. - `--sandbox-template-concurrent-workers` (1) - `--sandbox-warm-pool-max-batch-size` (300) - `--enable-warm-pool-eviction` (`true`) +- `--sandbox-warm-pool-disable-cr-management` (`false`) — the warm-pool + controller only reports pool status and creates no Sandbox CRs. Set it when + the L3 aggregated apiserver owns warm capacity, so the two do not both + provision - `--kube-api-qps` (200) — a negative value disables client-side rate limiting entirely, which also makes `--kube-api-burst` meaningless - `--kube-api-burst` (400) @@ -53,6 +67,11 @@ manages both externally. - `--enable-tracing` (`false`) - `--enable-pprof` (`false`) - `--enable-pprof-debug` (`false`) +- `--pprof-block-profile-rate` (1000000) — goroutine block profiling rate, + applied only with `--enable-pprof-debug` +- `--pprof-mutex-profile-fraction` (10) — mutex contention sampling, applied + only with `--enable-pprof-debug` +- `--version` — print the build identity and exit Use `--enable-pprof-debug` only in controlled environments because it exposes process details and enables block/mutex sampling. diff --git a/docs/e2b-compat.md b/docs/e2b-compat.md index 12b2c77..3444ab6 100644 --- a/docs/e2b-compat.md +++ b/docs/e2b-compat.md @@ -56,7 +56,7 @@ const sandbox = await Sandbox.create('registry.example.com/rt:24.04') |---|---|---| | `POST /sandboxes` | `store.Claim` | `templateID` → pool template; `timeout` → the claim's TTL (15s when omitted); `allow_internet_access` → `egress` lane, else the hardened `none` lane. `201` on success, `503` when the pool is drained (retryable). | | `GET /sandboxes`, `GET /v2/sandboxes` | `store.List` | Live sandboxes in the compat namespace. | -| `GET /sandboxes/{id}` | `store.List` + id match | `404` when no live sandbox carries the id. | +| `GET /sandboxes/{id}` | `store.GetByClaimID` | Resolves the owning node and materializes only that entry; `404` when no live sandbox carries the id. | | `DELETE /sandboxes/{id}` | `store.Release` | Releases the node-local claim id, never by Kubernetes name. `204`. | | `POST /sandboxes/{id}/timeout` | existence check | TTL is fixed by the node at claim time; the call is verified and acknowledged, not silently faked. | | `POST /sandboxes/{id}/refreshes` | existence check | Verifies that the sandbox is still live; it does not extend or refresh the node-owned deadline. | @@ -84,9 +84,18 @@ const sandbox = await Sandbox.create('registry.example.com/rt:24.04') `memUsed`, and `memTotal` come from the owning node when available; `cpuUsedPct`, `memCache`, `diskUsed`, and `diskTotal` are reported as zero. - **List/detail schema fields are compatibility values.** `startedAt` uses the - synthesized Sandbox creation time; `endAt` is `startedAt + 15s`, not the - owning node's authoritative deadline. `cpuCount`, `memoryMB`, and - `diskSizeMB` are reported as zero on these responses. + synthesized Sandbox creation time; `endAt` is the node-granted deadline when + the owning node published one, and `startedAt + 15s` otherwise. `cpuCount`, + `memoryMB`, and `diskSizeMB` are reported as zero on these responses. +- **`envdAccessToken` is returned only at claim time.** `POST /sandboxes` and + `POST /sandboxes/{id}/fork` carry the token the node just issued. The read + paths (`GET /sandboxes`, `GET /sandboxes/{id}`, `POST /sandboxes/{id}/connect`) + report it empty: node inventory deliberately carries no per-sandbox secret, + so a reconnecting client must keep the token from its create response. +- **`templateID` on read paths comes from node inventory.** The owning node + publishes the pool template with each entry; a node that does not yet publish + it makes `GET /sandboxes` and `GET /sandboxes/{id}` report an empty + `templateID`. `POST /sandboxes` always echoes the requested one. - **Size class** is pinned (`small`) — e2b's `NewSandbox` carries no size selector. - `metadata`, `envVars`, `autoPause`, and `secure` are accepted so SDK calls do diff --git a/docs/scaling-design.md b/docs/scaling-design.md index 76e4113..c100f8d 100644 --- a/docs/scaling-design.md +++ b/docs/scaling-design.md @@ -181,7 +181,7 @@ type NodeInventory struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` Node string `json:"node"` - Entries []InventoryEntry `json:"entries"` // {name, id, phase, claimRef, addr, deadline} + Entries []InventoryEntry `json:"entries"` // {name, id, phase, template, claimRef, addr, deadline} } ``` @@ -213,10 +213,12 @@ keeps the bias toward warm capacity while spreading a burst across the fleet, and a stale pick still costs at most one gossip redirect inside sandboxd, so correctness is unchanged. -The watch path makes the same trade in the other direction. Re-deriving the -fleet view is `O(nodes × sandboxes)` — 40 ms at 200 nodes and 50k sandboxes — -so a watch with nothing to report stretches its poll up to 8× the base interval -and snaps back to it on the first observed change. +The watch path makes the opposite trade. Re-deriving the fleet view is +`O(nodes × sandboxes)` — measured at 124 ms for the list and 173 ms for a full +poll tick at 200 nodes and 50k sandboxes — yet the poll cadence is fixed on +purpose: a widened interval would let a sandbox created and deleted inside the +gap produce neither an Added nor a Deleted event. One watcher per fleet is the +supported shape. ### L3 remaining follow-up: read after write without published inventory @@ -235,21 +237,21 @@ lifecycle verb issued inside that window answers `404`. Callers work around it by polling until visible — which is what `examples/lifecycle` does — so "create, then immediately pause" costs half a minute of polling. -**It is still `O(total inventory entries)` CPU on a miss.** The fast path avoids -materializing unrelated `Sandbox` objects, but finding the owner without an -index still compares entries across the fleet. That scan, rather than heap -growth from full-list synthesis, is the remaining scale constraint. +**A first lookup is still `O(total inventory entries)` CPU.** An owning-node +index now answers a repeat lookup from one node's inventory: measured at 200 +nodes × 2000 sandboxes, `Get` fell from 2.49 ms / 36 MB to 39 µs / 217 KB per +call. A first lookup, or one whose index entry was evicted, still compares +entries across the fleet. Both fall out of the same omission: `Claim` already returns the node and the claim id — the e2b create response even hands the node back to the client as `clientID` — and a lifecycle verb needs nothing else. The plan keeps that routing information instead of re-deriving it: -- **A. Claim-time index.** Record `sandboxID → (node, claimID)` when the claim - is made, consulted before the read view. Bound it with an LRU so memory is a - fixed budget rather than a function of load: measured 206 B/entry, so a - 200 k-entry cap is ~31 MB. Steady-state occupancy is `claim rate × TTL`, far - below the cap — 1 M sandboxes averaging a 5-minute life is ~117 k entries. +- **A. Claim-time index.** Implemented: the store records the owning node when + a claim is made and when a sweep resolves one, and consults it before fanning + out. It is bounded by generation swap rather than per-entry recency + bookkeeping, so memory is a fixed budget rather than a function of load. - **B. Authoritative fan-out on a miss.** A different replica, or an evicted entry, falls back to asking the nodes directly — the authoritative route the risk table already prescribes. Bounded by node count, off the read path for diff --git a/extensions/api/v1alpha1/sandboxclaim_conversion.go b/extensions/api/v1alpha1/sandboxclaim_conversion.go index 1c46bf5..da61c42 100644 --- a/extensions/api/v1alpha1/sandboxclaim_conversion.go +++ b/extensions/api/v1alpha1/sandboxclaim_conversion.go @@ -50,7 +50,7 @@ func (s *SandboxClaim) ConvertTo(dstRaw conversion.Hub) error { if raw, ok := s.Annotations[v1beta1SandboxClaimVolumeClaimTemplatesAnnotation]; ok { var vcts []sandboxv1beta1.PersistentVolumeClaimTemplate if err := json.Unmarshal([]byte(raw), &vcts); err != nil { - return fmt.Errorf("failed to unmarshal v1beta1 SandboxClaim volumeClaimTemplates: %w", err) + return fmt.Errorf("unmarshal v1beta1 SandboxClaim volumeClaimTemplates: %w", err) } dst.Spec.VolumeClaimTemplates = vcts if dst.Annotations != nil { @@ -79,7 +79,7 @@ func (s *SandboxClaim) ConvertFrom(srcRaw conversion.Hub) error { if src.Spec.VolumeClaimTemplates != nil { raw, err := json.Marshal(src.Spec.VolumeClaimTemplates) if err != nil { - return fmt.Errorf("failed to marshal v1beta1 SandboxClaim volumeClaimTemplates: %w", err) + return fmt.Errorf("marshal v1beta1 SandboxClaim volumeClaimTemplates: %w", err) } if s.Annotations == nil { s.Annotations = make(map[string]string) @@ -104,7 +104,7 @@ func restoreV1alpha1Spec(s *SandboxClaim, src *v1beta1.SandboxClaim) error { var original SandboxClaim if err := json.Unmarshal([]byte(stateJSON), &original); err != nil { - return fmt.Errorf("failed to unmarshal v1alpha1 SandboxClaim state: %w", err) + return fmt.Errorf("unmarshal v1alpha1 SandboxClaim state: %w", err) } // The template ref is kept either way: when the hub's warm pool changed there @@ -137,8 +137,8 @@ func isWarmPoolRefMatching(actualName, expectedName, sandboxName string) bool { } func stripRandomSuffix(name string) string { - if idx := strings.LastIndex(name, "-"); idx != -1 { - return name[:idx] + if head, _, ok := strings.CutLast(name, "-"); ok { + return head } return name } @@ -154,20 +154,16 @@ func convertClaimSpecTo(src *SandboxClaimSpec, dst *v1beta1.SandboxClaimSpec, cl dst.Lifecycle = nil } - // WarmPool / TemplateRef -> WarmPoolRef if src.WarmPool != nil && src.WarmPool.IsSpecificPool() { dst.WarmPoolRef = v1beta1.SandboxWarmPoolRef{ Name: string(*src.WarmPool), } } else { - // none or default warm pool policy if sandboxName != "" && claimName != sandboxName { - // Warm start dst.WarmPoolRef = v1beta1.SandboxWarmPoolRef{ Name: stripRandomSuffix(sandboxName), } } else { - // Cold start or no sandbox created yet dst.WarmPoolRef = v1beta1.SandboxWarmPoolRef{ Name: fmt.Sprintf("shadow-pool-%s", src.TemplateRef.Name), } @@ -201,7 +197,6 @@ func convertClaimSpecFrom(src *v1beta1.SandboxClaimSpec, dst *SandboxClaimSpec) dst.Lifecycle = nil } - // WarmPoolRef -> WarmPool / TemplateRef if templateName, ok := strings.CutPrefix(src.WarmPoolRef.Name, "shadow-pool-"); ok { dst.TemplateRef = SandboxTemplateRef{ Name: templateName, diff --git a/extensions/api/v1alpha1/sandboxclaim_conversion_test.go b/extensions/api/v1alpha1/sandboxclaim_conversion_test.go index 2ef2360..d0404b7 100644 --- a/extensions/api/v1alpha1/sandboxclaim_conversion_test.go +++ b/extensions/api/v1alpha1/sandboxclaim_conversion_test.go @@ -93,7 +93,6 @@ func TestSandboxClaimConversion(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - // Create src v1alpha1 SandboxClaim wpPolicy := WarmPoolPolicy(tc.warmPool) src := &SandboxClaim{ Name: tc.claimName, @@ -120,13 +119,11 @@ func TestSandboxClaimConversion(t *testing.T) { src.Spec.WarmPool = &wpPolicy } - // Convert to Hub (v1beta1) dst := &v1beta1.SandboxClaim{} if err := src.ConvertTo(dst); err != nil { t.Fatalf("failed to convert to v1beta1: %v", err) } - // Verify src annotations and labels were not mutated during ConvertTo if val, ok := src.Annotations[v1alpha1SandboxClaimStateAnnotation]; !ok || val != "some-old-state" { t.Errorf("src.Annotations was mutated during ConvertTo! expected 'some-old-state', got %q", val) } @@ -137,7 +134,6 @@ func TestSandboxClaimConversion(t *testing.T) { t.Errorf("expected 1 label in src, got %d", len(src.Labels)) } - // Verify the marshaled state in dst does not contain the state annotation itself (no nesting) marshaledState := dst.Annotations[v1alpha1SandboxClaimStateAnnotation] var stateObj SandboxClaim if err := json.Unmarshal([]byte(marshaledState), &stateObj); err != nil { @@ -147,23 +143,19 @@ func TestSandboxClaimConversion(t *testing.T) { t.Errorf("dst.Annotations state nestedly contains the state annotation! causing exponential growth") } - // Verify WarmPoolRef name in v1beta1 if dst.Spec.WarmPoolRef.Name != tc.expectedWarmPoolRef { t.Errorf("expected WarmPoolRef.Name %q, got %q", tc.expectedWarmPoolRef, dst.Spec.WarmPoolRef.Name) } - // Convert back to Spoke (v1alpha1) roundTrip := &SandboxClaim{} if err := roundTrip.ConvertFrom(dst); err != nil { t.Fatalf("failed to convert back to v1alpha1: %v", err) } - // Verify state annotation was stripped during ConvertFrom if _, ok := roundTrip.Annotations[v1alpha1SandboxClaimStateAnnotation]; ok { t.Errorf("roundTrip.Annotations still contains the state annotation after ConvertFrom!") } - // Verify round-trip preserves fields losslessly (due to state annotation preservation) if roundTrip.Spec.TemplateRef.Name != src.Spec.TemplateRef.Name { t.Errorf("roundtrip TemplateRef mismatch: expected %q, got %q", src.Spec.TemplateRef.Name, roundTrip.Spec.TemplateRef.Name) } @@ -181,7 +173,6 @@ func TestSandboxClaimConversion(t *testing.T) { } func TestSandboxClaimConversionFromHub(t *testing.T) { - // Test conversion of a v1beta1 SandboxClaim created without v1alpha1 state annotation (e.g. created directly via v1beta1 API) tests := []struct { name string warmPoolRefName string @@ -238,9 +229,6 @@ func TestSandboxClaimConversionFromHub(t *testing.T) { } } -// TestSandboxClaimVolumeClaimTemplatesRoundTrip asserts the v1beta1-only -// spec.volumeClaimTemplates survives a v1beta1 -> v1alpha1 -> v1beta1 round trip. -// v1alpha1 SandboxClaim has no such field, so it must be carried in an annotation. func TestSandboxClaimVolumeClaimTemplatesRoundTrip(t *testing.T) { vcts := []sandboxv1beta1.PersistentVolumeClaimTemplate{ { @@ -257,7 +245,6 @@ func TestSandboxClaimVolumeClaimTemplatesRoundTrip(t *testing.T) { }, } - // v1beta1 -> v1alpha1: the field must be preserved in the annotation. alpha := &SandboxClaim{} if err := alpha.ConvertFrom(src); err != nil { t.Fatalf("ConvertFrom: %v", err) @@ -266,7 +253,6 @@ func TestSandboxClaimVolumeClaimTemplatesRoundTrip(t *testing.T) { t.Fatalf("volumeClaimTemplates annotation not set on v1alpha1 object") } - // v1alpha1 -> v1beta1: the field must be restored and the annotation stripped. dst := &v1beta1.SandboxClaim{} if err := alpha.ConvertTo(dst); err != nil { t.Fatalf("ConvertTo: %v", err) @@ -281,8 +267,6 @@ func TestSandboxClaimVolumeClaimTemplatesRoundTrip(t *testing.T) { } } -// TestSandboxClaimNoVolumeClaimTemplatesNoAnnotation asserts a claim without -// volumeClaimTemplates does not gain the preservation annotation. func TestSandboxClaimNoVolumeClaimTemplatesNoAnnotation(t *testing.T) { src := &v1beta1.SandboxClaim{ Name: "claim", Namespace: "ns", diff --git a/extensions/api/v1alpha1/sandboxclaim_types.go b/extensions/api/v1alpha1/sandboxclaim_types.go index 6d81087..309ff96 100644 --- a/extensions/api/v1alpha1/sandboxclaim_types.go +++ b/extensions/api/v1alpha1/sandboxclaim_types.go @@ -52,9 +52,6 @@ const ( ShutdownPolicyRetain ShutdownPolicy = "Retain" ) -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// Important: Run "make" to regenerate code after modifying this file - // WarmPoolPolicy describes the policy for using warm pools. // It can be one of the following: // - "none": Do not use any warm pool, always create fresh sandboxes @@ -194,7 +191,7 @@ type SandboxClaim struct { // +kubebuilder:object:root=true -// SandboxList contains a list of Sandbox. +// SandboxClaimList contains a list of SandboxClaim. type SandboxClaimList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/extensions/api/v1alpha1/sandboxtemplate_conversion_test.go b/extensions/api/v1alpha1/sandboxtemplate_conversion_test.go index 86b54ea..455405a 100644 --- a/extensions/api/v1alpha1/sandboxtemplate_conversion_test.go +++ b/extensions/api/v1alpha1/sandboxtemplate_conversion_test.go @@ -59,7 +59,7 @@ func TestSandboxTemplateConversion(t *testing.T) { }, NetworkPolicy: &NetworkPolicySpec{ Ingress: []networkingv1.NetworkPolicyIngressRule{ - {}, // empty ingress rule to verify it gets converted + {}, }, }, NetworkPolicyManagement: NetworkPolicyManagementManaged, @@ -68,13 +68,11 @@ func TestSandboxTemplateConversion(t *testing.T) { }, } - // Convert to Hub (v1beta1) dst := &v1beta1.SandboxTemplate{} if err := src.ConvertTo(dst); err != nil { t.Fatalf("failed to convert to v1beta1: %v", err) } - // Verify src annotations and labels were not mutated during ConvertTo if val, ok := src.Annotations[v1alpha1SandboxTemplateStateAnnotation]; !ok || val != "some-old-state" { t.Errorf("src.Annotations was mutated during ConvertTo! expected 'some-old-state', got %q", val) } @@ -89,7 +87,6 @@ func TestSandboxTemplateConversion(t *testing.T) { t.Errorf("dst.Annotations carries the v1alpha1 state stash; the conversion is lossless and must not persist a second copy") } - // Verify v1beta1 fields if dst.Spec.PodTemplate.Spec.Containers[0].Image != "my-image:latest" { t.Errorf("unexpected image: %s", dst.Spec.PodTemplate.Spec.Containers[0].Image) } @@ -97,18 +94,15 @@ func TestSandboxTemplateConversion(t *testing.T) { t.Errorf("unexpected EnvVarsInjectionPolicy: %s", dst.Spec.EnvVarsInjectionPolicy) } - // Convert back to Spoke (v1alpha1) roundTrip := &SandboxTemplate{} if err := roundTrip.ConvertFrom(dst); err != nil { t.Fatalf("failed to convert back to v1alpha1: %v", err) } - // Verify state annotation was stripped during ConvertFrom if _, ok := roundTrip.Annotations[v1alpha1SandboxTemplateStateAnnotation]; ok { t.Errorf("roundTrip.Annotations still contains the state annotation after ConvertFrom!") } - // Verify round-trip preserves all fields if roundTrip.Spec.PodTemplate.Spec.Containers[0].Image != src.Spec.PodTemplate.Spec.Containers[0].Image { t.Errorf("roundtrip PodTemplate Image mismatch: expected %q, got %q", src.Spec.PodTemplate.Spec.Containers[0].Image, roundTrip.Spec.PodTemplate.Spec.Containers[0].Image) } @@ -124,7 +118,6 @@ func TestSandboxTemplateConversion(t *testing.T) { } func TestSandboxTemplateVolumeClaimTemplatesPolicyConversion(t *testing.T) { - // 1. Create v1beta1 SandboxTemplate with VolumeClaimTemplatesPolicy: Allowed src := &v1beta1.SandboxTemplate{ Name: "my-template", Namespace: "default", @@ -133,31 +126,26 @@ func TestSandboxTemplateVolumeClaimTemplatesPolicyConversion(t *testing.T) { }, } - // 2. Convert to Spoke (v1alpha1) spoke := &SandboxTemplate{} if err := spoke.ConvertFrom(src); err != nil { t.Fatalf("failed to convert from v1beta1: %v", err) } - // Verify v1beta1 policy was preserved in annotations if val, ok := spoke.Annotations["api.agents.x-k8s.io/v1beta1-volume-claim-templates-policy"]; !ok || val != string(v1beta1.VolumeClaimTemplatesPolicyAllowed) { t.Errorf("expected annotation api.agents.x-k8s.io/v1beta1-volume-claim-templates-policy to be 'Allowed', got %q", val) } - // 3. Convert back to Hub (v1beta1) dst := &v1beta1.SandboxTemplate{} if err := spoke.ConvertTo(dst); err != nil { t.Fatalf("failed to convert to v1beta1: %v", err) } - // Verify policy was perfectly restored if dst.Spec.VolumeClaimTemplatesPolicy != v1beta1.VolumeClaimTemplatesPolicyAllowed { t.Errorf("roundtrip VolumeClaimTemplatesPolicy mismatch: expected %q, got %q", v1beta1.VolumeClaimTemplatesPolicyAllowed, dst.Spec.VolumeClaimTemplatesPolicy) } } func TestSandboxTemplateVolumeClaimTemplatesPolicyStaleAnnotationClearing(t *testing.T) { - // 1. Create v1alpha1 SandboxTemplate with a stale policy annotation spoke := &SandboxTemplate{ Name: "stale-template", Namespace: "default", @@ -166,38 +154,31 @@ func TestSandboxTemplateVolumeClaimTemplatesPolicyStaleAnnotationClearing(t *tes }, } - // 2. Convert to Hub (v1beta1) dst := &v1beta1.SandboxTemplate{} if err := spoke.ConvertTo(dst); err != nil { t.Fatalf("failed to convert to v1beta1: %v", err) } - // Verify policy was restored to Allowed if dst.Spec.VolumeClaimTemplatesPolicy != v1beta1.VolumeClaimTemplatesPolicyAllowed { t.Fatalf("expected VolumeClaimTemplatesPolicy Allowed, got %q", dst.Spec.VolumeClaimTemplatesPolicy) } - // 3. Simulate user clearing the policy in v1beta1 dst.Spec.VolumeClaimTemplatesPolicy = "" - // 4. Convert back to Spoke (v1alpha1) spokeCleared := &SandboxTemplate{} if err := spokeCleared.ConvertFrom(dst); err != nil { t.Fatalf("failed to convert from v1beta1: %v", err) } - // Verify stale annotation was deleted if val, ok := spokeCleared.Annotations["api.agents.x-k8s.io/v1beta1-volume-claim-templates-policy"]; ok { t.Errorf("expected stale annotation to be deleted, but it remained with value %q", val) } - // 5. Convert back to Hub (v1beta1) again dstFinal := &v1beta1.SandboxTemplate{} if err := spokeCleared.ConvertTo(dstFinal); err != nil { t.Fatalf("failed to convert to v1beta1 final: %v", err) } - // Verify policy remains empty (not resurrected) if dstFinal.Spec.VolumeClaimTemplatesPolicy != "" { t.Errorf("expected VolumeClaimTemplatesPolicy to remain empty, got %q", dstFinal.Spec.VolumeClaimTemplatesPolicy) } diff --git a/extensions/api/v1alpha1/sandboxtemplate_types.go b/extensions/api/v1alpha1/sandboxtemplate_types.go index 6358913..5d9d731 100644 --- a/extensions/api/v1alpha1/sandboxtemplate_types.go +++ b/extensions/api/v1alpha1/sandboxtemplate_types.go @@ -22,16 +22,6 @@ import ( sandboxv1alpha1 "github.com/cocoonstack/sandbox-operator/api/v1alpha1" ) -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// Important: Run "make" to regenerate code after modifying this file - -// NetworkPolicyManagement defines whether the controller automatically generates -// and manages a shared NetworkPolicy for this template. -type NetworkPolicyManagement string - -// EnvVarsInjectionPolicy defines whether a SandboxClaim is allowed to inject or override environment variables. -type EnvVarsInjectionPolicy string - const ( // SandboxIDLabel is the label key applied to the Pod to identify the owning Claim UID. // The SandboxClaim controller injects this label into the Pod @@ -57,6 +47,13 @@ const ( EnvVarsInjectionPolicyDisallowed EnvVarsInjectionPolicy = "Disallowed" ) +// NetworkPolicyManagement defines whether the controller automatically generates +// and manages a shared NetworkPolicy for this template. +type NetworkPolicyManagement string + +// EnvVarsInjectionPolicy defines whether a SandboxClaim is allowed to inject or override environment variables. +type EnvVarsInjectionPolicy string + // NetworkPolicySpec defines the desired state of the NetworkPolicy. type NetworkPolicySpec struct { // ingress is a list of ingress rules to be applied to the sandbox. diff --git a/extensions/api/v1alpha1/sandboxwarmpool_conversion_test.go b/extensions/api/v1alpha1/sandboxwarmpool_conversion_test.go index 5159adc..0a4a2d2 100644 --- a/extensions/api/v1alpha1/sandboxwarmpool_conversion_test.go +++ b/extensions/api/v1alpha1/sandboxwarmpool_conversion_test.go @@ -32,7 +32,6 @@ func TestSandboxWarmPoolConversion(t *testing.T) { }, }, { - // Exercises the nil branch of convertWarmPoolSpecTo/convertWarmPoolSpecFrom. name: "nil update strategy", updateStrategy: nil, }, @@ -64,13 +63,11 @@ func TestSandboxWarmPoolConversion(t *testing.T) { }, } - // Convert to Hub (v1beta1) dst := &v1beta1.SandboxWarmPool{} if err := src.ConvertTo(dst); err != nil { t.Fatalf("failed to convert to v1beta1: %v", err) } - // Verify src annotations and labels were not mutated during ConvertTo if val, ok := src.Annotations[v1alpha1SandboxWarmPoolStateAnnotation]; !ok || val != "some-old-state" { t.Errorf("src.Annotations was mutated during ConvertTo! expected 'some-old-state', got %q", val) } @@ -85,7 +82,6 @@ func TestSandboxWarmPoolConversion(t *testing.T) { t.Errorf("dst.Annotations carries the v1alpha1 state stash; the conversion is lossless and must not persist a second copy") } - // Verify v1beta1 fields if dst.Spec.Replicas == nil || *dst.Spec.Replicas != 3 { t.Errorf("unexpected replicas: %v", dst.Spec.Replicas) } @@ -103,18 +99,15 @@ func TestSandboxWarmPoolConversion(t *testing.T) { t.Errorf("unexpected ready replicas: %d", dst.Status.ReadyReplicas) } - // Convert back to Spoke (v1alpha1) roundTrip := &SandboxWarmPool{} if err := roundTrip.ConvertFrom(dst); err != nil { t.Fatalf("failed to convert back to v1alpha1: %v", err) } - // Verify state annotation was stripped during ConvertFrom if _, ok := roundTrip.Annotations[v1alpha1SandboxWarmPoolStateAnnotation]; ok { t.Errorf("roundTrip.Annotations still contains the state annotation after ConvertFrom!") } - // Verify round-trip preserves all fields if roundTrip.Spec.Replicas != src.Spec.Replicas { t.Errorf("roundtrip Replicas mismatch: expected %d, got %d", src.Spec.Replicas, roundTrip.Spec.Replicas) } diff --git a/extensions/api/v1alpha1/sandboxwarmpool_types.go b/extensions/api/v1alpha1/sandboxwarmpool_types.go index b702fee..85cafe8 100644 --- a/extensions/api/v1alpha1/sandboxwarmpool_types.go +++ b/extensions/api/v1alpha1/sandboxwarmpool_types.go @@ -32,9 +32,6 @@ const ( OnReplenishSandboxWarmPoolUpdateStrategyType SandboxWarmPoolUpdateStrategyType = "OnReplenish" ) -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// Important: Run "make" to regenerate code after modifying this file - // SandboxWarmPoolSpec defines the desired state of SandboxWarmPool. type SandboxWarmPoolSpec struct { // replicas is the desired number of sandboxes in the pool. diff --git a/extensions/api/v1beta1/nodeinventory_types.go b/extensions/api/v1beta1/nodeinventory_types.go index d2f1beb..fef867f 100644 --- a/extensions/api/v1beta1/nodeinventory_types.go +++ b/extensions/api/v1beta1/nodeinventory_types.go @@ -38,6 +38,11 @@ type InventoryEntry struct { ID string `json:"id,omitempty"` // phase is the node-reported sandbox phase (e.g. Running). Phase string `json:"phase"` + // template is the pool template (base image) the sandbox was claimed from. + // It is the only place the aggregated read path can recover it: no + // per-sandbox object holds the pod spec. + // +optional + Template string `json:"template,omitempty"` // claimRef is the "/" of the SandboxClaim the sandbox is // bound to, if any. // +optional diff --git a/extensions/api/v1beta1/sandboxclaim_conversion.go b/extensions/api/v1beta1/sandboxclaim_conversion.go index 1ad0fda..5f48ec0 100644 --- a/extensions/api/v1beta1/sandboxclaim_conversion.go +++ b/extensions/api/v1beta1/sandboxclaim_conversion.go @@ -14,5 +14,4 @@ package v1beta1 -// Hub marks SandboxClaim as a conversion Hub. func (*SandboxClaim) Hub() {} diff --git a/extensions/api/v1beta1/sandboxclaim_types.go b/extensions/api/v1beta1/sandboxclaim_types.go index 16ac307..08838e1 100644 --- a/extensions/api/v1beta1/sandboxclaim_types.go +++ b/extensions/api/v1beta1/sandboxclaim_types.go @@ -46,9 +46,6 @@ const ( ShutdownPolicyRetain ShutdownPolicy = "Retain" ) -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// Important: Run "make" to regenerate code after modifying this file - // ShutdownPolicy describes the policy for shutting down the underlying Sandbox when the SandboxClaim expires. // +kubebuilder:validation:Enum=Delete;DeleteForeground;Retain type ShutdownPolicy string diff --git a/extensions/api/v1beta1/sandboxtemplate_conversion.go b/extensions/api/v1beta1/sandboxtemplate_conversion.go index e1c161e..4e6d1b1 100644 --- a/extensions/api/v1beta1/sandboxtemplate_conversion.go +++ b/extensions/api/v1beta1/sandboxtemplate_conversion.go @@ -14,5 +14,4 @@ package v1beta1 -// Hub marks SandboxTemplate as a conversion Hub. func (*SandboxTemplate) Hub() {} diff --git a/extensions/api/v1beta1/sandboxtemplate_types.go b/extensions/api/v1beta1/sandboxtemplate_types.go index 6059130..1717094 100644 --- a/extensions/api/v1beta1/sandboxtemplate_types.go +++ b/extensions/api/v1beta1/sandboxtemplate_types.go @@ -56,9 +56,6 @@ const ( VolumeClaimTemplatesPolicyOverrides VolumeClaimTemplatesPolicy = "Overrides" ) -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// Important: Run "make" to regenerate code after modifying this file - // NetworkPolicyManagement defines whether the controller automatically generates // and manages a shared NetworkPolicy for this template. type NetworkPolicyManagement string diff --git a/extensions/api/v1beta1/sandboxwarmpool_conversion.go b/extensions/api/v1beta1/sandboxwarmpool_conversion.go index e93f6f6..73060b6 100644 --- a/extensions/api/v1beta1/sandboxwarmpool_conversion.go +++ b/extensions/api/v1beta1/sandboxwarmpool_conversion.go @@ -14,5 +14,4 @@ package v1beta1 -// Hub marks SandboxWarmPool as a conversion Hub. func (*SandboxWarmPool) Hub() {} diff --git a/extensions/api/v1beta1/sandboxwarmpool_types.go b/extensions/api/v1beta1/sandboxwarmpool_types.go index c9f975a..9b703da 100644 --- a/extensions/api/v1beta1/sandboxwarmpool_types.go +++ b/extensions/api/v1beta1/sandboxwarmpool_types.go @@ -32,9 +32,6 @@ const ( OnReplenishSandboxWarmPoolUpdateStrategyType SandboxWarmPoolUpdateStrategyType = "OnReplenish" ) -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// Important: Run "make" to regenerate code after modifying this file - // SandboxTemplateRef references a SandboxTemplate. type SandboxTemplateRef struct { // name of the SandboxTemplate diff --git a/extensions/controllers/sandboxclaim_bench_test.go b/extensions/controllers/sandboxclaim_bench_test.go index 2aa4cd2..df6d669 100644 --- a/extensions/controllers/sandboxclaim_bench_test.go +++ b/extensions/controllers/sandboxclaim_bench_test.go @@ -10,7 +10,6 @@ import ( extensionsv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" ) -// A pool mid-operation: most claims already bound, a tail still waiting. const ( benchClaimTotal = 2500 benchClaimUnbound = 50 diff --git a/extensions/controllers/sandboxclaim_concurrent_exclusivity_test.go b/extensions/controllers/sandboxclaim_concurrent_exclusivity_test.go index 3b2090e..2aeb648 100644 --- a/extensions/controllers/sandboxclaim_concurrent_exclusivity_test.go +++ b/extensions/controllers/sandboxclaim_concurrent_exclusivity_test.go @@ -18,27 +18,18 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extensionsv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" - "github.com/cocoonstack/sandbox-operator/extensions/controllers/queue" "github.com/cocoonstack/sandbox-operator/internal/hash" asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/internal/queue" ) -// TestWarmPoolConcurrentClaimExclusivity is the L1 fast-path intent test: under -// many claims racing a shared warm pool concurrently, the pod-exclusivity -// invariant (#127) must hold — each warm Sandbox is adopted by at most one claim, -// each claim owns at most one Sandbox, and every warm Sandbox is consumed exactly -// once. The exclusivity guard is the in-memory WarmSandboxQueue: each candidate -// key is popped exactly once under its mutex, so two concurrent claims can never -// select the same warm Sandbox. This complements the sequential -// TestWarmPoolPodExclusivity by exercising the guard under real goroutine -// contention, the condition the decentralized claim fast-path is built for. func TestWarmPoolConcurrentClaimExclusivity(t *testing.T) { scheme := newScheme(t) ctx := t.Context() const ( warmCount = 12 - claimCount = 24 // more claims than warm sandboxes: the surplus must cold-start, never double-adopt + claimCount = 24 ) poolHash := hash.Name("pool") @@ -118,8 +109,6 @@ func TestWarmPoolConcurrentClaimExclusivity(t *testing.T) { MaxConcurrentReconciles: claimCount, } - // Fire every claim concurrently; each goroutine drives its own claim to Bound - // (bounded requeue passes) so the race is on adoption, not on scheduling. var wg sync.WaitGroup for _, cl := range claims { wg.Add(1) @@ -128,7 +117,7 @@ func TestWarmPoolConcurrentClaimExclusivity(t *testing.T) { req := reconcile.Request{Name: name, Namespace: "default"} for range 10 { if _, err := reconciler.Reconcile(ctx, req); err != nil { - continue // transient optimistic-concurrency conflict: retry + continue } cur := &extensionsv1beta1.SandboxClaim{} if err := fc.Get(ctx, req.NamespacedName, cur); err == nil && cur.Status.SandboxStatus.Name != "" { @@ -139,7 +128,6 @@ func TestWarmPoolConcurrentClaimExclusivity(t *testing.T) { } wg.Wait() - // Build sandbox -> owning claims across every Sandbox in the namespace. var allSandboxes sandboxv1beta1.SandboxList require.NoError(t, fc.List(ctx, &allSandboxes, client.InNamespace("default"))) @@ -156,13 +144,11 @@ func TestWarmPoolConcurrentClaimExclusivity(t *testing.T) { } } - // Invariant 1: no Sandbox is controlled by more than one claim. for sbName, owners := range sandboxToOwners { require.LessOrEqual(t, len(owners), 1, "sandbox %s adopted by multiple claims %v — pod-exclusivity violated under concurrency", sbName, owners) } - // Invariant 2: each claim owns at most one Sandbox. claimToSandbox := make(map[string][]string) for sbName, owners := range sandboxToOwners { for _, owner := range owners { @@ -174,11 +160,8 @@ func TestWarmPoolConcurrentClaimExclusivity(t *testing.T) { "claim %s owns multiple sandboxes %v", cl.Name, claimToSandbox[cl.Name]) } - // Invariant 3: every warm Sandbox was consumed exactly once (the queue is drained, - // none left double-adoptable). With claimCount > warmCount, all warm are adopted. require.Len(t, warmAdopted, warmCount, "expected all %d warm sandboxes adopted exactly once, got %d: %v", warmCount, len(warmAdopted), warmAdopted) - _, ok := testQueue.Get(npn) - require.False(t, ok, "warm queue must be fully drained after all warm sandboxes are claimed") + require.Zero(t, testQueue.Len(npn), "warm queue must be fully drained after all warm sandboxes are claimed") } diff --git a/extensions/controllers/sandboxclaim_controller.go b/extensions/controllers/sandboxclaim_controller.go index a303f76..ad6565b 100644 --- a/extensions/controllers/sandboxclaim_controller.go +++ b/extensions/controllers/sandboxclaim_controller.go @@ -15,6 +15,7 @@ package controllers import ( + "cmp" "context" "errors" "fmt" @@ -48,9 +49,9 @@ import ( v1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extensionsv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" - "github.com/cocoonstack/sandbox-operator/extensions/controllers/queue" "github.com/cocoonstack/sandbox-operator/internal/lifecycle" asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/internal/queue" ) const ( @@ -111,6 +112,11 @@ type triggeredAdoptionEntry struct { sandbox string } +type nodeSpread struct { + count int + first int +} + // failure is the reason/message pair behind a not-Ready claim. type failure struct { reason string @@ -282,7 +288,7 @@ func (r *SandboxClaimReconciler) SetupWithManager(mgr ctrl.Manager, concurrentWo return ctrl.NewControllerManagedBy(mgr). For(&extensionsv1beta1.SandboxClaim{}, builder.WithPredicates(r.getTimingPredicate())). - Owns(&v1beta1.Sandbox{}). + Owns(&v1beta1.Sandbox{}, builder.WithPredicates(claimSandboxChangePredicate())). Watches(&v1beta1.Sandbox{}, &sandboxEventHandler{sandboxQueue: r.WarmSandboxQueue}). Watches(&extensionsv1beta1.SandboxWarmPool{}, &warmPoolEventHandler{sandboxQueue: r.WarmSandboxQueue}). Watches( @@ -409,7 +415,6 @@ func (r *SandboxClaimReconciler) syncAdoptedSandboxMetadata(ctx context.Context, return nil } - patch := client.MergeFrom(sandbox.DeepCopy()) var mergedMeta v1beta1.PodMetadata template.Spec.PodTemplate.ObjectMeta.DeepCopyInto(&mergedMeta) if mergedMeta.Labels == nil { @@ -427,16 +432,21 @@ func (r *SandboxClaimReconciler) syncAdoptedSandboxMetadata(ctx context.Context, return err } - needsUpdate := !equality.Semantic.DeepEqual(&mergedMeta, &sandbox.Spec.PodTemplate.ObjectMeta) - if sandbox.Labels == nil { - sandbox.Labels = make(map[string]string) - } - needsUpdate = setOrDeleteLabel(sandbox.Labels, sandboxTemplateRefHash, templateHash) || needsUpdate - needsUpdate = setOrDeleteLabel(sandbox.Labels, v1beta1.CreatedByLabel, createdBy) || needsUpdate + // Compared before any mutation so the steady state costs no deep copy. + needsUpdate := !equality.Semantic.DeepEqual(&mergedMeta, &sandbox.Spec.PodTemplate.ObjectMeta) || + sandbox.Labels[sandboxTemplateRefHash] != templateHash || + sandbox.Labels[v1beta1.CreatedByLabel] != createdBy if !needsUpdate { return nil } + patch := client.MergeFrom(sandbox.DeepCopy()) + if sandbox.Labels == nil { + sandbox.Labels = make(map[string]string) + } + setOrDeleteLabel(sandbox.Labels, sandboxTemplateRefHash, templateHash) + setOrDeleteLabel(sandbox.Labels, v1beta1.CreatedByLabel, createdBy) + logger.V(1).Info("Updating sandbox metadata to match claim", "claim", claim.Name, "sandbox", sandbox.Name) sandbox.Spec.PodTemplate.ObjectMeta = mergedMeta if err := r.Patch(ctx, sandbox, patch); err != nil { @@ -460,7 +470,6 @@ func (r *SandboxClaimReconciler) reconcileActive(ctx context.Context, claim *ext logger := log.FromContext(ctx) logger.V(1).Info("Reconciling active claim", "claim", claim.Name) - // Upfront validation of additional metadata to skip unnecessary processing if err := r.validateAdditionalPodMetadata(&claim.Spec.AdditionalPodMetadata); err != nil { return nil, fmt.Errorf("%w: %w", ErrInvalidMetadata, err) } @@ -523,18 +532,8 @@ func (r *SandboxClaimReconciler) reconcileExpired(ctx context.Context, claim *ex func (r *SandboxClaimReconciler) updateStatus(ctx context.Context, oldStatus *extensionsv1beta1.SandboxClaimStatus, claim *extensionsv1beta1.SandboxClaim) error { logger := log.FromContext(ctx) - slices.SortFunc(oldStatus.Conditions, func(a, b metav1.Condition) int { - if a.Type < b.Type { - return -1 - } - return 1 - }) - slices.SortFunc(claim.Status.Conditions, func(a, b metav1.Condition) int { - if a.Type < b.Type { - return -1 - } - return 1 - }) + slices.SortFunc(oldStatus.Conditions, func(a, b metav1.Condition) int { return cmp.Compare(a.Type, b.Type) }) + slices.SortFunc(claim.Status.Conditions, func(a, b metav1.Condition) int { return cmp.Compare(a.Type, b.Type) }) if equality.Semantic.DeepEqual(oldStatus, &claim.Status) { return nil @@ -652,56 +651,8 @@ func (r *SandboxClaimReconciler) getCandidate(ctx context.Context, claim *extens } }() - pickSmart := func(keys []queue.SandboxKey) (queue.SandboxKey, bool) { - namespaceKeys := keys - - if len(namespaceKeys) == 0 { - return queue.SandboxKey{}, false - } - if len(namespaceKeys) == 1 { - return namespaceKeys[0], true - } - - var scheduledKeys []queue.SandboxKey - var unscheduledKeys []queue.SandboxKey - for _, key := range namespaceKeys { - if key.NodeName != "" { - scheduledKeys = append(scheduledKeys, key) - } else { - unscheduledKeys = append(unscheduledKeys, key) - } - } - - // NodeSpread: the node with the most remaining warm sandboxes has been - // picked the least, so drain it first. - if len(scheduledKeys) > 0 { - nodeCounts := make(map[string]int) - for _, key := range scheduledKeys { - nodeCounts[key.NodeName]++ - } - - maxCount := 0 - for _, count := range nodeCounts { - if count > maxCount { - maxCount = count - } - } - - var bestCandidates []queue.SandboxKey - for _, key := range scheduledKeys { - if nodeCounts[key.NodeName] == maxCount { - bestCandidates = append(bestCandidates, key) - } - } - - return bestCandidates[0], true - } - - return unscheduledKeys[0], true - } - for { - adoptedKey, ok := r.WarmSandboxQueue.GetWithStrategy(namespacedWarmPoolName, pickSmart) + adoptedKey, ok := r.WarmSandboxQueue.GetWithStrategy(namespacedWarmPoolName, pickNodeSpread) if !ok { if fallbackSandbox != nil { adoptingFallback = true @@ -744,7 +695,6 @@ func (r *SandboxClaimReconciler) adoptSandboxFromCandidates(ctx context.Context, logger := log.FromContext(ctx) namespacedWarmPoolNameForQueue := queue.GetNamespacedWarmPoolName(claim.Namespace, claim.Spec.WarmPoolRef.Name) - // Keep trying until we successfully adopt a sandbox, or run out of candidates for range 3 { adopted, adoptedKey, err := r.getCandidate(ctx, claim) if err != nil { @@ -825,17 +775,15 @@ func (r *SandboxClaimReconciler) tryAdopt(ctx context.Context, claim *extensions } func (r *SandboxClaimReconciler) completeAdoption(ctx context.Context, claim *extensionsv1beta1.SandboxClaim, adopted *v1beta1.Sandbox) error { - // Take a snapshot of the sandbox BEFORE we mutate it to generate a clean JSON Patch. originalAdopted := adopted.DeepCopy() templateHash := adopted.Labels[sandboxTemplateRefHash] - // Remove warm pool labels so the sandbox no longer appears in warm pool queries - delete(adopted.Labels, warmPoolSandboxLabel) - delete(adopted.Labels, v1beta1.SandboxTemplateHashLabel) if adopted.Labels == nil { adopted.Labels = make(map[string]string) } + delete(adopted.Labels, warmPoolSandboxLabel) + delete(adopted.Labels, v1beta1.SandboxTemplateHashLabel) adopted.Labels[v1beta1.SandboxLaunchTypeLabel] = v1beta1.SandboxLaunchTypeWarm // Remove the warm pool's default eviction annotation so the adopted sandbox // is protected from autoscaler scale-downs now that it hosts active state. @@ -1086,7 +1034,6 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten } } } else { - // Validate the VolumeClaimTemplates from the SandboxTemplate. if err := validateVolumeClaimTemplates(template.Spec.VolumeClaimTemplates); err != nil { return nil, fmt.Errorf("invalid volume claim templates in template: %w", err) } @@ -1111,7 +1058,6 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten return nil, err } - // Apply secure defaults to the sandbox pod spec ApplySandboxSecureDefaults(template, &sandbox.Spec.PodTemplate.Spec) if err := controllerutil.SetControllerReference(claim, sandbox, r.Scheme); err != nil { @@ -1424,23 +1370,19 @@ func (r *SandboxClaimReconciler) resolveTemplateName(sandbox *v1beta1.Sandbox) s func (r *SandboxClaimReconciler) getOrRecordObservedTime(obj client.Object) time.Time { key := types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()} - // Fast path: Entry already exists and UID matches if entry, ok := r.observedTimes.Load(key); ok { if entry.uid == obj.GetUID() { return entry.timestamp } } - // Slow path: Entry missing or UID mismatched newEntry := observedTimeEntry{timestamp: time.Now(), uid: obj.GetUID()} actual, loaded := r.observedTimes.LoadOrStore(key, newEntry) if loaded { - // Handle concurrent insertion: check if we need to overwrite due to UID mismatch if actual.uid != obj.GetUID() { r.observedTimes.Store(key, newEntry) return newEntry.timestamp } - // UID matches, return the loaded timestamp return actual.timestamp } return newEntry.timestamp @@ -1589,16 +1531,6 @@ func (r *SandboxClaimReconciler) recordCreationLatencyMetric( r.recordSandboxCreationLatency(sandbox, launchType, templateName) } -func hasSandboxExpiredCondition(conditions []metav1.Condition) bool { - readyCondition := meta.FindStatusCondition(conditions, string(v1beta1.SandboxConditionReady)) - return readyCondition != nil && readyCondition.Reason == v1beta1.SandboxReasonExpired -} - -func hasClaimExpiredCondition(conditions []metav1.Condition) bool { - readyCondition := meta.FindStatusCondition(conditions, string(v1beta1.SandboxConditionReady)) - return readyCondition != nil && readyCondition.Reason == extensionsv1beta1.ClaimExpiredReason -} - // sandboxEventHandler implements handler.EventHandler for the SandboxClaimReconciler. type sandboxEventHandler struct { sandboxQueue *queue.SimpleSandboxQueue @@ -1630,7 +1562,6 @@ func (h *sandboxEventHandler) Update(ctx context.Context, e event.UpdateEvent, _ nodeScheduled := oldSandbox.Status.NodeName != newSandbox.Status.NodeName if (!oldAdoptable && newAdoptable) || (newAdoptable && poolChanged) || (newAdoptable && nodeScheduled) { - // Add/update sandbox in the queue key := queue.SandboxKey{ Namespace: newSandbox.Namespace, Name: newSandbox.Name, @@ -1663,50 +1594,12 @@ func (h *sandboxEventHandler) Delete(ctx context.Context, e event.DeleteEvent, _ namespacedWarmPoolName := queue.GetNamespacedWarmPoolName(sandbox.Namespace, warmPoolName) - // Actively delete the Ghost Pod from the memory queue logger := log.FromContext(ctx) logger.V(1).Info("Removing deleted sandbox from warm pool queue", "namespace", sandbox.Namespace, "sandbox", key) h.sandboxQueue.RemoveItem(namespacedWarmPoolName, key) } } -func verifySandboxCandidate(candidate *v1beta1.Sandbox, claim *extensionsv1beta1.SandboxClaim) error { - if candidate.Namespace != claim.Namespace { - return fmt.Errorf("%w: sandbox is in %q, claim is in %q", ErrCrossNamespaceAdoption, candidate.Namespace, claim.Namespace) - } - - if err := isAdoptable(candidate); err != nil { - return err - } - - warmPoolName := getWarmPoolName(candidate) - if warmPoolName == "" || warmPoolName != claim.Spec.WarmPoolRef.Name { - return fmt.Errorf("incorrect warm pool, expected %v", claim.Spec.WarmPoolRef.Name) - } - return nil -} - -func isAdoptable(candidate *v1beta1.Sandbox) error { - if !candidate.DeletionTimestamp.IsZero() { - return fmt.Errorf("sandbox is deleted") - } - if _, ok := candidate.Labels[warmPoolSandboxLabel]; !ok { - return fmt.Errorf("sandbox is missing the warm pool sandbox label") - } - if _, ok := candidate.Labels[sandboxTemplateRefHash]; !ok { - return fmt.Errorf("sandbox is missing the sandbox template ref hash label") - } - - controllerRef := metav1.GetControllerOf(candidate) - if controllerRef == nil { - return fmt.Errorf("sandbox %s/%s is unowned and cannot be safely adopted", candidate.Namespace, candidate.Name) - } - if controllerRef.APIVersion != extensionsv1beta1.GroupVersion.String() || controllerRef.Kind != warmPoolKind { - return fmt.Errorf("sandbox %s/%s is not managed by warm pool. Controller: %v", candidate.Namespace, candidate.Name, controllerRef) - } - return nil -} - type warmPoolEventHandler struct { sandboxQueue *queue.SimpleSandboxQueue } @@ -1730,31 +1623,9 @@ func (h *warmPoolEventHandler) Delete(ctx context.Context, e event.DeleteEvent, logger := log.FromContext(ctx) logger.Info("SandboxWarmPool deleted, cleaning up memory queue", "namespace", warmPool.Namespace, "warmPool", warmPool.Name) - // Actively drop the entire queue from memory h.sandboxQueue.RemoveQueue(namespacedWarmPoolName) } -func getWarmPoolName(obj metav1.Object) string { - if ctrl := metav1.GetControllerOf(obj); ctrl != nil && ctrl.Kind == warmPoolKind { - return ctrl.Name - } - for _, ref := range obj.GetOwnerReferences() { - if ref.Kind == warmPoolKind { - return ref.Name - } - } - return "" -} - -func shouldSuppressError(err error) bool { - for _, target := range suppressErrors { - if errors.Is(err, target) { - return true - } - } - return false -} - // observedTimeMap is a type-safe wrapper around sync.Map that only stores observedTimeEntry values. type observedTimeMap struct { inner sync.Map @@ -1803,6 +1674,74 @@ func (m *triggeredAdoptionMap) Delete(key types.NamespacedName) { m.inner.Delete(key) } +func hasSandboxExpiredCondition(conditions []metav1.Condition) bool { + readyCondition := meta.FindStatusCondition(conditions, string(v1beta1.SandboxConditionReady)) + return readyCondition != nil && readyCondition.Reason == v1beta1.SandboxReasonExpired +} + +func hasClaimExpiredCondition(conditions []metav1.Condition) bool { + readyCondition := meta.FindStatusCondition(conditions, string(v1beta1.SandboxConditionReady)) + return readyCondition != nil && readyCondition.Reason == extensionsv1beta1.ClaimExpiredReason +} + +func verifySandboxCandidate(candidate *v1beta1.Sandbox, claim *extensionsv1beta1.SandboxClaim) error { + if candidate.Namespace != claim.Namespace { + return fmt.Errorf("%w: sandbox is in %q, claim is in %q", ErrCrossNamespaceAdoption, candidate.Namespace, claim.Namespace) + } + + if err := isAdoptable(candidate); err != nil { + return err + } + + warmPoolName := getWarmPoolName(candidate) + if warmPoolName == "" || warmPoolName != claim.Spec.WarmPoolRef.Name { + return fmt.Errorf("incorrect warm pool, expected %v", claim.Spec.WarmPoolRef.Name) + } + return nil +} + +func isAdoptable(candidate *v1beta1.Sandbox) error { + if !candidate.DeletionTimestamp.IsZero() { + return fmt.Errorf("sandbox is deleted") + } + if _, ok := candidate.Labels[warmPoolSandboxLabel]; !ok { + return fmt.Errorf("sandbox is missing the warm pool sandbox label") + } + if _, ok := candidate.Labels[sandboxTemplateRefHash]; !ok { + return fmt.Errorf("sandbox is missing the sandbox template ref hash label") + } + + controllerRef := metav1.GetControllerOf(candidate) + if controllerRef == nil { + return fmt.Errorf("sandbox %s/%s is unowned and cannot be safely adopted", candidate.Namespace, candidate.Name) + } + if controllerRef.APIVersion != extensionsv1beta1.GroupVersion.String() || controllerRef.Kind != warmPoolKind { + return fmt.Errorf("sandbox %s/%s is not managed by warm pool. Controller: %v", candidate.Namespace, candidate.Name, controllerRef) + } + return nil +} + +func getWarmPoolName(obj metav1.Object) string { + if ctrl := metav1.GetControllerOf(obj); ctrl != nil && ctrl.Kind == warmPoolKind { + return ctrl.Name + } + for _, ref := range obj.GetOwnerReferences() { + if ref.Kind == warmPoolKind { + return ref.Name + } + } + return "" +} + +func shouldSuppressError(err error) bool { + for _, target := range suppressErrors { + if errors.Is(err, target) { + return true + } + } + return false +} + // soonerRequeue returns the earlier of a pending requeue and a proposed delay, // treating a zero pending requeue as "none scheduled". func soonerRequeue(pending, proposed time.Duration) time.Duration { @@ -1823,19 +1762,40 @@ func stagedAnnotationsSurvived(claim *extensionsv1beta1.SandboxClaim, staged map return true } +func pickNodeSpread(keys []queue.SandboxKey) (queue.SandboxKey, bool) { + if len(keys) == 0 { + return queue.SandboxKey{}, false + } + nodes := make(map[string]nodeSpread, len(keys)) + best := nodeSpread{first: -1} + for i, key := range keys { + if key.NodeName == "" { + continue + } + seen := nodes[key.NodeName] + if seen.count == 0 { + seen.first = i + } + seen.count++ + nodes[key.NodeName] = seen + if seen.count > best.count || (seen.count == best.count && seen.first < best.first) { + best = seen + } + } + if best.count == 0 { + return keys[0], true + } + return keys[best.first], true +} + // setOrDeleteLabel forces labels[key] to want, removing the entry when want is -// empty. It reports whether labels changed. -func setOrDeleteLabel(labels map[string]string, key, want string) bool { +// empty. +func setOrDeleteLabel(labels map[string]string, key, want string) { if want == "" { - _, existed := labels[key] delete(labels, key) - return existed - } - if labels[key] == want { - return false + return } labels[key] = want - return true } // notReady builds the Ready=False condition for f. @@ -1894,6 +1854,37 @@ func ensureClaimIdentityLabels(labels map[string]string, claim *extensionsv1beta return labels } +// claimSandboxChangePredicate passes the owned-Sandbox transitions a claim +// mirrors: pod IPs, the Ready and Finished conditions, deletion, and the spec or +// label edits the metadata sync repairs. Other status writes (node name, service, +// label selector) would re-reconcile the claim for nothing. +func claimSandboxChangePredicate() predicate.Funcs { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldSb, okOld := e.ObjectOld.(*v1beta1.Sandbox) + newSb, okNew := e.ObjectNew.(*v1beta1.Sandbox) + if !okOld || !okNew { + return true + } + return oldSb.Generation != newSb.Generation || + oldSb.DeletionTimestamp.IsZero() != newSb.DeletionTimestamp.IsZero() || + !maps.Equal(oldSb.Labels, newSb.Labels) || + !slices.Equal(oldSb.Status.PodIPs, newSb.Status.PodIPs) || + sandboxConditionChanged(oldSb, newSb, string(v1beta1.SandboxConditionReady)) || + sandboxConditionChanged(oldSb, newSb, string(v1beta1.SandboxConditionFinished)) + }, + } +} + +func sandboxConditionChanged(oldSb, newSb *v1beta1.Sandbox, conditionType string) bool { + oldCond := meta.FindStatusCondition(oldSb.Status.Conditions, conditionType) + newCond := meta.FindStatusCondition(newSb.Status.Conditions, conditionType) + if oldCond == nil || newCond == nil { + return oldCond != newCond + } + return *oldCond != *newCond +} + // isSandboxReady checks if a sandbox has Ready=True condition. func isSandboxReady(sb *v1beta1.Sandbox) bool { for _, cond := range sb.Status.Conditions { @@ -1936,7 +1927,6 @@ func mergeVolumeClaimTemplates( return nil, ErrVolumeClaimTemplatesDisallowed case extensionsv1beta1.VolumeClaimTemplatesPolicyAllowed: - // Check for any overrides (name match) templateMap := make(map[string]struct{}, len(templateVCTs)) for _, vct := range templateVCTs { templateMap[vct.Name] = struct{}{} @@ -1949,14 +1939,12 @@ func mergeVolumeClaimTemplates( return slices.Concat(templateVCTs, claimVCTs), nil case extensionsv1beta1.VolumeClaimTemplatesPolicyOverrides: - // Merge by Name: claim VCT replaces template VCT by name if they match, and new ones are appended. merged := make([]v1beta1.PersistentVolumeClaimTemplate, 0, len(templateVCTs)+len(claimVCTs)) claimMap := make(map[string]v1beta1.PersistentVolumeClaimTemplate, len(claimVCTs)) for _, vct := range claimVCTs { claimMap[vct.Name] = vct } - // Keep template VCTs unless overridden by name for _, vct := range templateVCTs { if override, ok := claimMap[vct.Name]; ok { merged = append(merged, override) @@ -1966,7 +1954,6 @@ func mergeVolumeClaimTemplates( } } - // Append any new volume templates introduced by the claim for _, vct := range claimVCTs { if _, exists := claimMap[vct.Name]; exists { merged = append(merged, vct) diff --git a/extensions/controllers/sandboxclaim_controller_test.go b/extensions/controllers/sandboxclaim_controller_test.go index 9efda4d..2f94a8f 100644 --- a/extensions/controllers/sandboxclaim_controller_test.go +++ b/extensions/controllers/sandboxclaim_controller_test.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -46,9 +45,9 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extensionsv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" - "github.com/cocoonstack/sandbox-operator/extensions/controllers/queue" "github.com/cocoonstack/sandbox-operator/internal/hash" asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/internal/queue" ) func TestSandboxClaimReconcile(t *testing.T) { @@ -409,7 +408,6 @@ func TestSandboxClaimReconcile(t *testing.T) { }} readySandbox.Status.PodIPs = []string{"10.244.0.6"} - // Validation Functions validateSandboxHasDefaultAutomountToken := func(t *testing.T, sandbox *sandboxv1beta1.Sandbox, template *extensionsv1beta1.SandboxTemplate) { expectedSpec := template.Spec.PodTemplate.Spec.DeepCopy() expectedSpec.AutomountServiceAccountToken = new(false) @@ -430,7 +428,6 @@ func TestSandboxClaimReconcile(t *testing.T) { } validateSandboxDNSUntouched := func(t *testing.T, sandbox *sandboxv1beta1.Sandbox, _ *extensionsv1beta1.SandboxTemplate) { - // Prove that the air-gapped fix works: DNS should not be overridden! if sandbox.Spec.PodTemplate.Spec.DNSPolicy == corev1.DNSNone { t.Errorf("Expected DNSPolicy to remain untouched, but it was set to None") } @@ -572,7 +569,6 @@ func TestSandboxClaimReconcile(t *testing.T) { t.Errorf("expected Sandbox metadata to have label %q=%q, got %q", sandboxTemplateRefHash, expectedHash, val) } - // Verify DNS Bypass is successfully injected if sandbox.Spec.PodTemplate.Spec.DNSPolicy != corev1.DNSNone { t.Errorf("Expected DNSPolicy to be 'None', got %q", sandbox.Spec.PodTemplate.Spec.DNSPolicy) } @@ -995,10 +991,9 @@ func TestSandboxClaimReconcile(t *testing.T) { t.Run(tc.name, func(t *testing.T) { scheme := newScheme(t) - // Logic to determine which claim to use (Default to 'claim' if nil) claimToUse := tc.claimToReconcile if claimToUse == nil { - claimToUse = claim // Fallback for older tests + claimToUse = claim } allObjects := append(slices.Clone(tc.existingObjects), claimToUse) @@ -1013,7 +1008,6 @@ func TestSandboxClaimReconcile(t *testing.T) { AllowedLabelDomains: tc.allowedDomains, } - // Pre-populate PodQueue with any existing pods for _, obj := range allObjects { if sb, ok := obj.(*sandboxv1beta1.Sandbox); ok { if isAdoptable(sb) != nil { @@ -1046,7 +1040,7 @@ func TestSandboxClaimReconcile(t *testing.T) { } if tc.expectSandbox { - // Verify the controller injected the template hash label so the NP can find the pod + templateName := sandbox.Annotations[sandboxv1beta1.SandboxTemplateRefAnnotation] if templateName == "" { t.Fatalf("expected sandbox to have template ref annotation, but it was missing") @@ -1090,8 +1084,6 @@ func TestSandboxClaimReconcile(t *testing.T) { } } -// TestSandboxClaimCleanupPolicy verifies that the Claim deletes itself -// based on its own timestamp, and deletes the Sandbox if Policy=Retain. func TestSandboxClaimCleanupPolicy(t *testing.T) { template := &extensionsv1beta1.SandboxTemplate{ Name: "cleanup-template", Namespace: "default", @@ -1117,7 +1109,6 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { } } - // Helper to create a Sandbox. createSandbox := func(claimName string, isExpired bool) *sandboxv1beta1.Sandbox { reason := "SandboxReady" status := metav1.ConditionTrue @@ -1150,10 +1141,10 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { claim *extensionsv1beta1.SandboxClaim sandboxIsExpired bool isWarmPool bool - sandboxNotOwned bool // sandbox exists at statusName but belongs to a different owner + sandboxNotOwned bool expectClaimDeleted bool expectSandboxDeleted bool - expectSandboxStatusCleared bool // SandboxStatus.Name and PodIPs must be empty + expectSandboxStatusCleared bool expectStatus string }{ { @@ -1161,7 +1152,7 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { claim: createClaim("retain-claim", extensionsv1beta1.ShutdownPolicyRetain), sandboxIsExpired: false, expectClaimDeleted: false, - expectSandboxDeleted: true, // Controller explicitly deletes Sandbox here. + expectSandboxDeleted: true, expectStatus: extensionsv1beta1.ClaimExpiredReason, }, { @@ -1170,7 +1161,7 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { sandboxIsExpired: false, isWarmPool: true, expectClaimDeleted: false, - expectSandboxDeleted: true, // Controller explicitly deletes Sandbox here. + expectSandboxDeleted: true, expectStatus: extensionsv1beta1.ClaimExpiredReason, }, { @@ -1187,9 +1178,7 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { claim: createClaim("delete-claim-synced", extensionsv1beta1.ShutdownPolicyDelete), sandboxIsExpired: true, expectClaimDeleted: true, - // In unit tests (FakeClient), deleting the Parent (Claim) does NOT automatically delete the Child (Sandbox). - // Since our controller only deletes the Claim and relies on K8s GC for the Sandbox, - // the Sandbox will technically remain in the FakeClient. This is expected behavior for tests. + expectSandboxDeleted: false, expectStatus: "", }, @@ -1198,7 +1187,7 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { claim: createClaim("delete-claim-race", extensionsv1beta1.ShutdownPolicyDelete), sandboxIsExpired: false, expectClaimDeleted: true, - expectSandboxDeleted: false, // Same as above: FakeClient doesn't simulate GC. + expectSandboxDeleted: false, expectStatus: "", }, { @@ -1206,8 +1195,7 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { claim: createClaim("delete-fg-claim", extensionsv1beta1.ShutdownPolicyDeleteForeground), sandboxIsExpired: false, expectClaimDeleted: true, - // FakeClient doesn't simulate GC or foreground propagation, - // so the Sandbox will remain. The important thing is the Claim is deleted. + expectSandboxDeleted: false, expectStatus: "", }, @@ -1226,13 +1214,11 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { scheme := newScheme(t) sandbox := createSandbox(tc.claim.Name, tc.sandboxIsExpired) - // Hack: Simulate warmPool adopted sandbox if tc.isWarmPool { sandbox.Name = "warm-pool-sandbox-adopted" tc.claim.Status.SandboxStatus.Name = sandbox.Name } - // Simulate a sandbox that exists at statusName but belongs to a different owner. if tc.sandboxNotOwned { sandbox.Name = "foreign-sandbox" sandbox.OwnerReferences = []metav1.OwnerReference{ @@ -1262,7 +1248,6 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { } } - // 1. Verify Claim var fetchedClaim extensionsv1beta1.SandboxClaim err = client.Get(t.Context(), req.NamespacedName, &fetchedClaim) @@ -1274,7 +1259,7 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { if err != nil { t.Errorf("Expected Claim to exist, but got error: %v", err) } - // Verify Status Message for Retained Claims + foundReason := false for _, cond := range fetchedClaim.Status.Conditions { if cond.Type == string(sandboxv1beta1.SandboxConditionReady) && cond.Reason == tc.expectStatus { @@ -1295,10 +1280,8 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { } } - // 2. Verify Sandbox var fetchedSandbox sandboxv1beta1.Sandbox - // The Sandbox might now have different name than the claim! err = client.Get(t.Context(), types.NamespacedName{Name: sandbox.Name, Namespace: sandbox.Namespace}, &fetchedSandbox) if tc.expectSandboxDeleted { @@ -1306,8 +1289,6 @@ func TestSandboxClaimCleanupPolicy(t *testing.T) { t.Error("Expected Sandbox to be deleted (explicitly by controller), but it still exists") } } else { - // For Policy=Delete. - // We verify it still exists to ensure the controller didn't delete it explicitly (which would be redundant). if k8errors.IsNotFound(err) { t.Error("Expected Sandbox to persist (FakeClient has no GC), but it was deleted") } @@ -1614,7 +1595,6 @@ func TestSandboxClaimTTLCleanupRequiresPersistedExpiredStatus(t *testing.T) { require.True(t, k8errors.IsNotFound(err)) } -// TestSandboxProvisionEvent verifies that Sandbox creation emits "SandboxProvisioned". func TestSandboxProvisionEvent(t *testing.T) { scheme := newScheme(t) claimName := "provision-event-claim" @@ -1655,10 +1635,9 @@ func TestSandboxProvisionEvent(t *testing.T) { t.Fatalf("Reconcile failed: %v", err) } - // Verify 'SandboxProvisioned' Event expectedMsg := fmt.Sprintf("Normal SandboxProvisioned Created Sandbox %q", claimName) foundProvisionEvent := false - // Drain the channel + Loop: for { select { @@ -1736,7 +1715,6 @@ func TestCreateSandboxPropagatesVolumeClaimTemplates(t *testing.T) { t.Fatalf("Reconcile failed: %v", err) } - // Verify sandbox was created with volumeClaimTemplates sandbox := &sandboxv1beta1.Sandbox{} err = fakeClient.Get(t.Context(), types.NamespacedName{Name: claimName, Namespace: "default"}, sandbox) if err != nil { @@ -2030,7 +2008,7 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { expectSandboxAdoption: true, expectedAdoptedSandbox: "pool-sb-2", expectNewSandboxCreated: false, - simulateConflicts: 1, // Fail update on the first sandbox, succeed on the second + simulateConflicts: 1, }, { name: "preserves template eviction annotation false when adopting sandbox", @@ -2163,13 +2141,10 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { } } - // 1. Initialize the Queue warmSandboxQueue := queue.NewSimpleSandboxQueue() - // 2. Seed the Queue with the existing objects from the test case for _, obj := range tc.existingObjects { if sb, ok := obj.(*sandboxv1beta1.Sandbox); ok { - // Only add valid, adoptable sandboxes to the queue if isAdoptable(sb) == nil { warmPoolName := getWarmPoolName(sb) namespacedWarmPoolName := queue.GetNamespacedWarmPoolName(sb.Namespace, warmPoolName) @@ -2179,7 +2154,6 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { } } - // 3. Inject the seeded Queue into the Reconciler reconciler := &SandboxClaimReconciler{ Client: fakeClient, Scheme: scheme, @@ -2200,7 +2174,7 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { } if tc.expectSandboxAdoption { - // Verify the adopted sandbox has correct labels and owner reference + var adoptedSandbox sandboxv1beta1.Sandbox err = fakeClient.Get(ctx, types.NamespacedName{ Name: tc.expectedAdoptedSandbox, @@ -2210,7 +2184,6 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { t.Fatalf("failed to get adopted sandbox: %v", err) } - // 1. Verify warm pool labels were removed if _, exists := adoptedSandbox.Labels[warmPoolSandboxLabel]; exists { t.Errorf("expected warm pool label to be removed from adopted sandbox") } @@ -2222,7 +2195,6 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { t.Errorf("expected adopted sandbox to have launch type label %q, got %q; labels=%v", sandboxv1beta1.SandboxLaunchTypeWarm, val, adoptedSandbox.Labels) } - // Verify eviction annotation is either matched against expected value or removed by default if len(tc.expectedPodAnnotations) > 0 { for key, expected := range tc.expectedPodAnnotations { val, exists := adoptedSandbox.Spec.PodTemplate.ObjectMeta.Annotations[key] @@ -2238,19 +2210,16 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { } } - // 2. Verify SandboxID label was added to pod template expectedUID := string(types.UID("claim-uid")) if val := adoptedSandbox.Spec.PodTemplate.ObjectMeta.Labels[extensionsv1beta1.SandboxIDLabel]; val != expectedUID { t.Errorf("expected pod template to have SandboxID label %q, got %q", expectedUID, val) } - // 3. Verify claim is the controller owner controllerRef := metav1.GetControllerOf(&adoptedSandbox) if controllerRef == nil || controllerRef.UID != claim.UID { t.Errorf("expected adopted sandbox to be controlled by claim, got %v", controllerRef) } - // 4. Verify the adopted sandbox records the adopted pod name require.Equal(t, adoptedSandbox.Name, adoptedSandbox.Annotations[sandboxv1beta1.SandboxPodNameAnnotation]) for key, expected := range tc.expectedAnnotations { @@ -2265,7 +2234,6 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { require.Equal(t, expected, adoptedSandbox.Spec.PodTemplate.ObjectMeta.Labels[key]) } - // 5. Verify the claim records the assigned sandbox annotation var updatedClaim extensionsv1beta1.SandboxClaim if err := fakeClient.Get(ctx, req.NamespacedName, &updatedClaim); err != nil { t.Fatalf("failed to get updated claim: %v", err) @@ -2273,7 +2241,7 @@ func TestSandboxClaimSandboxAdoption(t *testing.T) { require.Equal(t, tc.expectedAdoptedSandbox, updatedClaim.Annotations[extensionsv1beta1.AssignedSandboxNameAnnotation]) } else if tc.expectNewSandboxCreated { - // Verify a new sandbox was created with the claim's name + var sandbox sandboxv1beta1.Sandbox err = fakeClient.Get(ctx, req.NamespacedName, &sandbox) if err != nil { @@ -2295,10 +2263,8 @@ func TestSandboxEventHandler_Delete_RemovesGhostPods(t *testing.T) { namespacedWarmPoolName := queue.GetNamespacedWarmPoolName("default", warmPoolName) key := queue.SandboxKey{Namespace: "default", Name: "ghost-pod"} - // 1. Add the pod to the queue q.Add(namespacedWarmPoolName, key) - // 2. Create the mock Sandbox object that is being deleted sb := &sandboxv1beta1.Sandbox{ Name: "ghost-pod", Namespace: "default", @@ -2308,12 +2274,9 @@ func TestSandboxEventHandler_Delete_RemovesGhostPods(t *testing.T) { }}, } - // 3. Fire the Delete event handler.Delete(t.Context(), event.DeleteEvent{Object: sb}, nil) - // 4. Verify the Ghost Pod was removed from the queue - _, ok := q.Get(namespacedWarmPoolName) - if ok { + if q.Len(namespacedWarmPoolName) != 0 { t.Errorf("Expected the deleted sandbox to be removed from the queue") } } @@ -2325,28 +2288,21 @@ func TestWarmPoolEventHandler_Delete_RemovesEntireQueue(t *testing.T) { warmPoolName := "old-warmpool" key := queue.SandboxKey{Namespace: "default", Name: "abandoned-pod"} - // 1. Add a pod to this warmpool's queue using namespace-aware index namespacedWarmPoolName := queue.GetNamespacedWarmPoolName("default", warmPoolName) q.Add(namespacedWarmPoolName, key) - // 2. Create the mock SandboxWarmPool object that is being deleted warmPool := &extensionsv1beta1.SandboxWarmPool{ Name: warmPoolName, Namespace: "default", } - // 3. Fire the Delete event handler.Delete(t.Context(), event.DeleteEvent{Object: warmPool}, nil) - // 4. Verify the entire queue was wiped out - _, ok := q.Get(namespacedWarmPoolName) - if ok { + if q.Len(namespacedWarmPoolName) != 0 { t.Errorf("Expected the entire queue to be removed when the warmpool was deleted") } } -// TestSandboxClaimNoReAdoption verifies that a second reconcile does not adopt another -// sandbox from the warm pool when the claim already owns one. func TestSandboxClaimNoReAdoption(t *testing.T) { scheme := newScheme(t) @@ -2368,7 +2324,6 @@ func TestSandboxClaimNoReAdoption(t *testing.T) { poolNameHash := hash.Name("test-pool") - // Claim that already adopted a sandbox (name recorded in status) claim := &extensionsv1beta1.SandboxClaim{ Name: "test-claim", Namespace: "default", UID: "claim-uid", Spec: extensionsv1beta1.SandboxClaimSpec{WarmPoolRef: extensionsv1beta1.SandboxWarmPoolRef{Name: "test-pool"}}, @@ -2377,7 +2332,6 @@ func TestSandboxClaimNoReAdoption(t *testing.T) { }, } - // The previously adopted sandbox (owned by claim, different name) adoptedSandbox := &sandboxv1beta1.Sandbox{ Name: "adopted-sb", Namespace: "default", OwnerReferences: []metav1.OwnerReference{{ @@ -2387,7 +2341,6 @@ func TestSandboxClaimNoReAdoption(t *testing.T) { Spec: sandboxv1beta1.SandboxSpec{SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}}}, OperatingMode: sandboxv1beta1.SandboxOperatingModeRunning}, } - // Another warm pool sandbox that should NOT be adopted poolSandbox := &sandboxv1beta1.Sandbox{ Name: "pool-sb-extra", Namespace: "default", Labels: map[string]string{ @@ -2424,7 +2377,6 @@ func TestSandboxClaimNoReAdoption(t *testing.T) { t.Fatalf("reconcile failed: %v", err) } - // Verify the pool sandbox was NOT adopted (still has warm pool labels) var extra sandboxv1beta1.Sandbox if err := fakeClient.Get(ctx, types.NamespacedName{Name: "pool-sb-extra", Namespace: "default"}, &extra); err != nil { t.Fatalf("failed to get pool sandbox: %v", err) @@ -2571,7 +2523,6 @@ func TestRecordCreationLatencyMetric(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Reset the metrics registry for a clean test asmetrics.ClaimStartupLatency.Reset() asmetrics.ClaimControllerStartupLatency.Reset() @@ -2586,7 +2537,6 @@ func TestRecordCreationLatencyMetric(t *testing.T) { r.recordCreationLatencyMetric(ctx, tc.claim, tc.oldStatus, tc.sandbox) - // Verify the metric was observed in the Prometheus registry count := testutil.CollectAndCount(asmetrics.ClaimStartupLatency) if count != tc.expectedObservations { t.Errorf("expected %d observations for ClaimStartupLatency, got %d", tc.expectedObservations, count) @@ -2640,13 +2590,11 @@ func TestSandboxClaimCreationMetric(t *testing.T) { t.Fatalf("reconcile failed: %v", err) } - // Verify metric val := testutil.ToFloat64(asmetrics.SandboxClaimCreationTotal.WithLabelValues("default", "test-template", asmetrics.LaunchTypeCold, "test-warmpool", "not_ready", "unknown")) if val != 1 { t.Errorf("expected metric count 1, got %v", val) } - // Verify created Sandbox labels are absent sb := &sandboxv1beta1.Sandbox{} if err := client.Get(t.Context(), types.NamespacedName{Name: claim.Name, Namespace: "default"}, sb); err != nil { t.Fatalf("failed to get created sandbox: %v", err) @@ -2662,7 +2610,6 @@ func TestSandboxClaimCreationMetric(t *testing.T) { t.Run("Warm Start", func(t *testing.T) { asmetrics.SandboxClaimCreationTotal.Reset() - // Create a warm pool sandbox poolNameHash := hash.Name("test-warmpool") warmSandbox := &sandboxv1beta1.Sandbox{ Name: "warm-sb", @@ -2715,13 +2662,11 @@ func TestSandboxClaimCreationMetric(t *testing.T) { t.Fatalf("reconcile failed: %v", err) } - // Verify metric val := testutil.ToFloat64(asmetrics.SandboxClaimCreationTotal.WithLabelValues("default", "test-template", asmetrics.LaunchTypeWarm, "test-warmpool", "ready", "unknown")) if val != 1 { t.Errorf("expected metric count 1, got %v", val) } - // Verify adopted Sandbox labels are removed (since claim lacks it) sb := &sandboxv1beta1.Sandbox{} if err := client.Get(t.Context(), types.NamespacedName{Name: "warm-sb", Namespace: "default"}, sb); err != nil { t.Fatalf("failed to get adopted sandbox: %v", err) @@ -2918,7 +2863,7 @@ func TestSandboxClaimTimingPredicates(t *testing.T) { r.observedTimes.Store(key, observedTimeEntry{timestamp: time.Now(), uid: "uid-2"}) }, trigger: func(p predicate.Predicate) bool { - return p.Delete(event.DeleteEvent{Object: claim1}) // claim1 has uid-1 + return p.Delete(event.DeleteEvent{Object: claim1}) }, verify: func(t *testing.T, r *SandboxClaimReconciler) { _, ok := r.observedTimes.Load(key) @@ -2933,7 +2878,7 @@ func TestSandboxClaimTimingPredicates(t *testing.T) { r.observedTimes.Store(key, observedTimeEntry{timestamp: time.Now(), uid: "uid-1"}) }, trigger: func(p predicate.Predicate) bool { - return p.Delete(event.DeleteEvent{Object: claim1}) // claim1 has uid-1 + return p.Delete(event.DeleteEvent{Object: claim1}) }, verify: func(t *testing.T, r *SandboxClaimReconciler) { _, ok := r.observedTimes.Load(key) @@ -2946,7 +2891,7 @@ func TestSandboxClaimTimingPredicates(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - r.observedTimes = observedTimeMap{} // Reset map for each test case + r.observedTimes = observedTimeMap{} if tc.setup != nil { tc.setup(r) } @@ -3011,7 +2956,6 @@ func TestGetOrRecordObservedTime(t *testing.T) { res := r.getOrRecordObservedTime(tc.claimToRecord) - // Verify map state for the recorded claim recordedKey := types.NamespacedName{Name: tc.claimToRecord.Name, Namespace: tc.claimToRecord.Namespace} entry, ok := r.observedTimes.Load(recordedKey) if !ok { @@ -3023,7 +2967,7 @@ func TestGetOrRecordObservedTime(t *testing.T) { } if tc.expectNewTimestamp { - // Expect a new timestamp + if entry.timestamp.IsZero() { t.Error("Expected timestamp to be set") } @@ -3034,7 +2978,7 @@ func TestGetOrRecordObservedTime(t *testing.T) { t.Error("Expected returned time to match stored time") } } else { - // Expect specific timestamp + if !entry.timestamp.Equal(tc.expectedReturnTime) { t.Errorf("Expected timestamp %v, got %v", tc.expectedReturnTime, entry.timestamp) } @@ -3125,7 +3069,6 @@ func TestSandboxClaimReconcileCleanup(t *testing.T) { wantEntries int }{ { - // Reconcile on a missing claim removes the observedTimes entry via the NotFound fallback. name: "NotFound reconcile removes stale entry", build: func(t *testing.T) (*SandboxClaimReconciler, []*extensionsv1beta1.SandboxClaim) { cl := makeReadyClaim("stale-claim") @@ -3140,8 +3083,6 @@ func TestSandboxClaimReconcileCleanup(t *testing.T) { wantEntries: 0, }, { - // CreateFunc adds an entry; recordControllerStartupLatency removes it on the first - // Not-Ready → Ready transition detected by recordCreationLatencyMetric. name: "new claim transitioning to Ready cleans its entry", build: func(t *testing.T) (*SandboxClaimReconciler, []*extensionsv1beta1.SandboxClaim) { cl := &extensionsv1beta1.SandboxClaim{ @@ -3161,8 +3102,6 @@ func TestSandboxClaimReconcileCleanup(t *testing.T) { wantEntries: 0, }, { - // Simulates a controller restart where the informer replays UpdateFunc for existing, - // already-Ready claims. The reconciler must correctly clean up the observed time entries. name: "already-Ready claims are cleaned up after restart simulation", build: func(t *testing.T) (*SandboxClaimReconciler, []*extensionsv1beta1.SandboxClaim) { const n = 10 @@ -3185,8 +3124,6 @@ func TestSandboxClaimReconcileCleanup(t *testing.T) { wantEntries: 0, }, { - // Simulates an update for already ready claim. - // The reconciler must correctly clean up the observed time entries. name: "post-Ready UpdateFunc re-creates entry that is then cleaned on next reconcile", build: func(t *testing.T) (*SandboxClaimReconciler, []*extensionsv1beta1.SandboxClaim) { cl := &extensionsv1beta1.SandboxClaim{ @@ -3198,24 +3135,22 @@ func TestSandboxClaimReconcileCleanup(t *testing.T) { }, action: func(t *testing.T, r *SandboxClaimReconciler, claims []*extensionsv1beta1.SandboxClaim) { pred := r.getTimingPredicate() - // Step 1: CreateFunc → entry added + for _, cl := range claims { pred.Create(event.CreateEvent{Object: cl}) } - // Step 2: First reconcile → Not-Ready → Ready transition → entry cleaned. + reconcileAll(t, r, claims) - // Step 3: Post-Ready UpdateFunc + for _, cl := range claims { pred.Update(event.UpdateEvent{ObjectOld: cl, ObjectNew: cl}) } - // Step 4: Reconcile with old=Ready, new=Ready + reconcileAll(t, r, claims) }, wantEntries: 0, }, { - // DeleteFunc is the sole cleanup path for entries that accumulated after a restart. - // Firing it for each claim fully drains the map. name: "DeleteFunc drains entries accumulated after restart simulation", build: func(t *testing.T) (*SandboxClaimReconciler, []*extensionsv1beta1.SandboxClaim) { const n = 10 @@ -3270,7 +3205,6 @@ func TestVerifySandboxCandidate_NamespaceIsolation(t *testing.T) { }, } - // 1. Valid Sandbox (Same Namespace) validSandbox := &sandboxv1beta1.Sandbox{ Name: "valid-sandbox", Namespace: "namespace-a", @@ -3282,11 +3216,10 @@ func TestVerifySandboxCandidate_NamespaceIsolation(t *testing.T) { APIVersion: extensionsv1beta1.GroupVersion.String(), Kind: "SandboxWarmPool", Name: "test-warmpool", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, } - // 2. Invalid Sandbox (Different Namespace, but identical hash) invalidSandbox := &sandboxv1beta1.Sandbox{ Name: "invalid-sandbox", Namespace: "namespace-b", @@ -3298,16 +3231,14 @@ func TestVerifySandboxCandidate_NamespaceIsolation(t *testing.T) { APIVersion: extensionsv1beta1.GroupVersion.String(), Kind: "SandboxWarmPool", Name: "test-warmpool", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, } - // Test Valid: Should return nil (no error) if err := verifySandboxCandidate(validSandbox, claim); err != nil { t.Errorf("Expected valid sandbox in the same namespace to be accepted, but got: %v", err) } - // Test Invalid: Should return an error about cross-namespace adoption err := verifySandboxCandidate(invalidSandbox, claim) if err == nil { t.Fatal("FATAL: Cross-namespace sandbox was successfully verified! The namespace check is missing.") @@ -3316,9 +3247,6 @@ func TestVerifySandboxCandidate_NamespaceIsolation(t *testing.T) { } } -// TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag verifies that during informer cache lag, -// the assigned sandbox annotation on the claim is used to identify the previously adopted Sandbox, -// preventing duplicate adoptions from the warm pool. func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { scheme := newScheme(t) @@ -3364,7 +3292,7 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "warmpool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{ @@ -3377,7 +3305,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { }, } - // Another sandbox in the warm pool that we want to make sure doesn't get adopted poolNameHash := hash.Name("test-pool") extraSandbox := &sandboxv1beta1.Sandbox{ Name: "pool-sb-extra", @@ -3391,7 +3318,7 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "warmpool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}}}}, Status: sandboxv1beta1.SandboxStatus{ @@ -3425,11 +3352,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { req := reconcile.Request{Name: "test-claim", Namespace: "default"} - // Run reconcile. Adoption is triggered on this pass, but the sandbox is not yet - // observed as controlled by the claim (cache lag), so the reconcile must NOT finalize - // the claim and must requeue to try again. Post-#1107 the requeue is a bounded - // fixed-delay requeue with a nil error (not an exponentially-rate-limited error), so - // the compounding backoff that inflated adoption tail latency no longer occurs. res, err := reconciler.Reconcile(t.Context(), req) if err != nil { t.Fatalf("Expected reconcile to requeue without error during cache lag, got error: %v", err) @@ -3438,7 +3360,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { t.Fatalf("Expected the bounded cache-lag requeue (%v), got RequeueAfter=%v", adoptionCacheLagRequeueDelay, res.RequeueAfter) } - // Verify that the claim status was NOT updated with the sandbox name (adoption deferred) updatedClaim := &extensionsv1beta1.SandboxClaim{} if err := fakeClient.Get(t.Context(), types.NamespacedName{Name: "test-claim", Namespace: "default"}, updatedClaim); err != nil { t.Fatalf("failed to get claim: %v", err) @@ -3448,8 +3369,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { t.Error("expected claim status to NOT be updated with 'adopted-sb' during cache lag") } - // The cache-lag retry is a benign signal: the Ready condition must report the - // specific AdoptionPending reason, not a generic reconciler failure. readyCondition := meta.FindStatusCondition(updatedClaim.Status.Conditions, string(sandboxv1beta1.SandboxConditionReady)) if readyCondition == nil { t.Fatal("expected Ready condition to be set during cache lag") @@ -3458,7 +3377,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { t.Errorf("expected Ready condition reason %q during cache lag, got %q (message: %q)", "AdoptionPending", readyCondition.Reason, readyCondition.Message) } - // Verify that the extra warm sandbox was NOT adopted (it should still have its warm pool labels) var extra sandboxv1beta1.Sandbox if err := fakeClient.Get(t.Context(), types.NamespacedName{Name: "pool-sb-extra", Namespace: "default"}, &extra); err != nil { t.Fatalf("failed to get extra warm sandbox: %v", err) @@ -3467,8 +3385,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { t.Error("expected extra warm sandbox to still have warm pool label, meaning it was not incorrectly adopted during cache lag") } - // Simulate the cache catching up! - // Fetch the adopted sandbox object, add the SandboxClaim owner reference, and update it in fakeClient. var adopted sandboxv1beta1.Sandbox if err := fakeClient.Get(t.Context(), types.NamespacedName{Name: "adopted-sb", Namespace: "default"}, &adopted); err != nil { t.Fatalf("failed to get adopted sandbox: %v", err) @@ -3478,19 +3394,17 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { Kind: "SandboxClaim", Name: "test-claim", UID: "claim-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }} if err := fakeClient.Update(t.Context(), &adopted); err != nil { t.Fatalf("failed to update adopted sandbox with claim owner ref: %v", err) } - // Run reconcile AGAIN _, err = reconciler.Reconcile(t.Context(), req) if err != nil { t.Fatalf("Expected second Reconcile to succeed after cache caught up, but failed: %v", err) } - // Verify that the claim status WAS updated this time! if err := fakeClient.Get(t.Context(), types.NamespacedName{Name: "test-claim", Namespace: "default"}, updatedClaim); err != nil { t.Fatalf("failed to get claim: %v", err) } @@ -3505,7 +3419,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { t.Errorf("expected assigned adopted sandbox to have launch type label %q, got %q; labels=%v", sandboxv1beta1.SandboxLaunchTypeWarm, val, adopted.Labels) } - // Verify that the extra warm sandbox was STILL NOT adopted (it should still have its warm pool labels) if err := fakeClient.Get(t.Context(), types.NamespacedName{Name: "pool-sb-extra", Namespace: "default"}, &extra); err != nil { t.Fatalf("failed to get extra warm sandbox: %v", err) } @@ -3514,10 +3427,6 @@ func TestSandboxClaimPreventsDuplicateAdoptionDuringCacheLag(t *testing.T) { } } -// TestSandboxClaimAdoptionCacheLagDoesNotRepatch verifies that while the informer cache -// keeps returning the stale (warm-pool-owned) view of an already-adopted sandbox, the -// bounded cache-lag requeues wait for convergence WITHOUT re-sending the adoption patch -// on every pass. func TestSandboxClaimAdoptionCacheLagDoesNotRepatch(t *testing.T) { scheme := newScheme(t) @@ -3563,7 +3472,7 @@ func TestSandboxClaimAdoptionCacheLagDoesNotRepatch(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "warmpool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{ @@ -3576,8 +3485,6 @@ func TestSandboxClaimAdoptionCacheLagDoesNotRepatch(t *testing.T) { }, } - // Frozen warm-pool-owned view: served on every Get to simulate an informer - // cache that has not converged yet, no matter what was patched. staleSandbox := adoptedSandbox.DeepCopy() sandboxPatches := 0 @@ -3611,7 +3518,6 @@ func TestSandboxClaimAdoptionCacheLagDoesNotRepatch(t *testing.T) { } req := reconcile.Request{Name: "test-claim", Namespace: "default"} - // Pass 1: adoption is triggered (patches the sandbox) and defers via bounded requeue. res, err := reconciler.Reconcile(t.Context(), req) if err != nil { t.Fatalf("pass 1: expected nil error, got: %v", err) @@ -3624,7 +3530,6 @@ func TestSandboxClaimAdoptionCacheLagDoesNotRepatch(t *testing.T) { t.Fatal("pass 1: expected the adoption patch to be sent") } - // Passes 2 and 3: cache still stale — must keep requeueing WITHOUT re-patching. for pass := 2; pass <= 3; pass++ { res, err = reconciler.Reconcile(t.Context(), req) if err != nil { @@ -3639,10 +3544,6 @@ func TestSandboxClaimAdoptionCacheLagDoesNotRepatch(t *testing.T) { } } -// TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus verifies that a claim whose -// status was already finalized with a sandbox (e.g. a controller restart racing a stale -// informer) does NOT have its SandboxStatus.Name/PodIPs wiped or its Ready condition -// downgraded by a benign cache-lag adoption retry pass. func TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus(t *testing.T) { scheme := newScheme(t) @@ -3656,7 +3557,7 @@ func TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus(t *testing.T) { Spec: extensionsv1beta1.SandboxClaimSpec{ WarmPoolRef: extensionsv1beta1.SandboxWarmPoolRef{Name: "test-pool"}, }, - // Status already finalized on a previous pass. + Status: extensionsv1beta1.SandboxClaimStatus{ Conditions: []metav1.Condition{{ Type: string(sandboxv1beta1.SandboxConditionReady), @@ -3702,7 +3603,7 @@ func TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "warmpool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{ @@ -3715,8 +3616,6 @@ func TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus(t *testing.T) { }, } - // Frozen warm-pool-owned view: served on every Get to simulate an informer - // cache that has not converged yet, no matter what was patched. staleSandbox := adoptedSandbox.DeepCopy() fakeClient := fake.NewClientBuilder(). @@ -3734,7 +3633,6 @@ func TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus(t *testing.T) { }). Build() - // Fresh reconciler (empty triggeredAdoptions), as after a controller restart. reconciler := &SandboxClaimReconciler{ Client: fakeClient, Scheme: scheme, @@ -3771,15 +3669,9 @@ func TestSandboxClaimAdoptionCacheLagPreservesFinalizedStatus(t *testing.T) { } } -// TestSandboxClaimFreshAdoptionDoesNotRepatchDuringCacheLag verifies that the primary -// warm-adoption entry point (adoptSandboxFromCandidates) records the completed adoption -// in the dedup cache, so the very next cache-lag pass waits via the bounded requeue -// without re-sending the adoption patch — and without wiping the status finalized on -// the adoption pass. func TestSandboxClaimFreshAdoptionDoesNotRepatchDuringCacheLag(t *testing.T) { scheme := newScheme(t) - // No assigned-sandbox annotation: adoption goes through the candidate queue. claim := &extensionsv1beta1.SandboxClaim{ Name: "test-claim", Namespace: "default", @@ -3818,7 +3710,7 @@ func TestSandboxClaimFreshAdoptionDoesNotRepatchDuringCacheLag(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "warmpool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}}}}, Status: sandboxv1beta1.SandboxStatus{ @@ -3828,8 +3720,6 @@ func TestSandboxClaimFreshAdoptionDoesNotRepatchDuringCacheLag(t *testing.T) { }, } - // Frozen warm-pool-owned view: served on every Get to simulate an informer - // cache that never converges within the test, no matter what was patched. staleSandbox := warmSandbox.DeepCopy() sandboxPatches := 0 @@ -3869,7 +3759,6 @@ func TestSandboxClaimFreshAdoptionDoesNotRepatchDuringCacheLag(t *testing.T) { } req := reconcile.Request{Name: "test-claim", Namespace: "default"} - // Pass 1: fresh adoption through adoptSandboxFromCandidates; status is finalized. if _, err := reconciler.Reconcile(t.Context(), req); err != nil { t.Fatalf("pass 1: expected nil error, got: %v", err) } @@ -3886,9 +3775,6 @@ func TestSandboxClaimFreshAdoptionDoesNotRepatchDuringCacheLag(t *testing.T) { t.Fatalf("pass 1: expected status to be finalized with 'warm-sb', got %q", updatedClaim.Status.SandboxStatus.Name) } - // Passes 2 and 3: cache still shows the warm-pool owner — must wait via the - // bounded requeue WITHOUT re-sending the adoption patch, and WITHOUT wiping - // the status finalized on pass 1. for pass := 2; pass <= 3; pass++ { res, err := reconciler.Reconcile(t.Context(), req) if err != nil { @@ -3953,7 +3839,7 @@ func TestSandboxClaimPreventsAdoptionFromWrongWarmPool(t *testing.T) { Kind: "SandboxWarmPool", Name: "wrong-pool", UID: "wrong-pool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "img"}}}}}}, Status: sandboxv1beta1.SandboxStatus{ @@ -4036,7 +3922,6 @@ func TestSandboxClaimRecoveryWhenTemplateCreated(t *testing.T) { Spec: extensionsv1beta1.SandboxWarmPoolSpec{TemplateRef: extensionsv1beta1.SandboxTemplateRef{Name: templateName}}, } - // Step 1: Reconcile without template fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(claim, warmPool). @@ -4053,7 +3938,6 @@ func TestSandboxClaimRecoveryWhenTemplateCreated(t *testing.T) { req := reconcile.Request{Name: claimName, Namespace: "default"} - // Should return no error but RequeueAfter because template is missing result, err := reconciler.Reconcile(t.Context(), req) if err != nil { t.Fatalf("expected no error when template is missing, but got %v", err) @@ -4062,7 +3946,6 @@ func TestSandboxClaimRecoveryWhenTemplateCreated(t *testing.T) { t.Errorf("expected RequeueAfter to be 1 minute, got %v", result.RequeueAfter) } - // Verify status is set to TemplateNotFound var updatedClaim extensionsv1beta1.SandboxClaim if err := fakeClient.Get(t.Context(), req.NamespacedName, &updatedClaim); err != nil { t.Fatalf("failed to get claim: %v", err) @@ -4072,7 +3955,6 @@ func TestSandboxClaimRecoveryWhenTemplateCreated(t *testing.T) { t.Errorf("expected status reason 'TemplateNotFound', got %v", cond) } - // Step 2: Create template and reconcile again if err := fakeClient.Create(t.Context(), template); err != nil { t.Fatalf("failed to create template: %v", err) } @@ -4082,7 +3964,6 @@ func TestSandboxClaimRecoveryWhenTemplateCreated(t *testing.T) { t.Fatalf("expected no error when template exists, but got %v", err) } - // Verify sandbox is created var sandbox sandboxv1beta1.Sandbox if err := fakeClient.Get(t.Context(), req.NamespacedName, &sandbox); err != nil { t.Fatalf("expected sandbox to be created, but got error: %v", err) @@ -4117,11 +3998,6 @@ func TestMapWarmPoolToClaims(t *testing.T) { Name: warmPoolName, Namespace: "default", } - // We need to manually set up the indexer on the fake client's indexer if it supports it, - // or we can mock the List behavior. Fake client from controller-runtime does NOT use indexers by default - // unless configured with WithIndex. - - // Let's use the WithIndex option on the fake client builder to support the matchingFields query! fakeClientWithIndex := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(claim1, claim2, claimOther, claimBound, warmPool). @@ -4151,11 +4027,9 @@ func TestMapWarmPoolToClaims(t *testing.T) { } func TestIsAdoptable_RejectsUnowned(t *testing.T) { - // 1. Create a warm pool template hash poolNameHash := hash.Name("test-pool") templateHash := hash.Name("test-template") - // 2. Mock an unowned Sandbox (no OwnerReferences) unownedSandbox := &sandboxv1beta1.Sandbox{ Name: "unowned-sandbox", Namespace: "default", @@ -4165,12 +4039,10 @@ func TestIsAdoptable_RejectsUnowned(t *testing.T) { }, } - // 3. Verify it is rejected err := isAdoptable(unownedSandbox) require.Error(t, err) require.Contains(t, err.Error(), "unowned") - // 4. Mock an owned Sandbox (pointing to SandboxWarmPool) ownedSandbox := unownedSandbox.DeepCopy() ownedSandbox.OwnerReferences = []metav1.OwnerReference{ { @@ -4178,15 +4050,13 @@ func TestIsAdoptable_RejectsUnowned(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "pool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }, } - // 5. Verify it is accepted err = isAdoptable(ownedSandbox) require.NoError(t, err) - // 6. Mock an owned Sandbox pointing to a different kind (e.g. SandboxClaim, which is NOT WarmPool) ownedByClaimSandbox := unownedSandbox.DeepCopy() ownedByClaimSandbox.OwnerReferences = []metav1.OwnerReference{ { @@ -4194,11 +4064,10 @@ func TestIsAdoptable_RejectsUnowned(t *testing.T) { Kind: "SandboxClaim", Name: "test-claim", UID: "claim-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }, } - // 7. Verify it is rejected err = isAdoptable(ownedByClaimSandbox) require.Error(t, err) require.Contains(t, err.Error(), "not managed by warm pool") @@ -4227,7 +4096,7 @@ func TestSandboxClaimAdoptionStrategy(t *testing.T) { Kind: "SandboxWarmPool", Name: "test-pool", UID: "warmpool-uid", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }, }, Spec: sandboxv1beta1.SandboxSpec{ @@ -4361,11 +4230,11 @@ func TestSandboxClaimAdoptionStrategy(t *testing.T) { require.NotNil(t, controllerRef) require.Equal(t, claim.UID, controllerRef.UID) - // Verify that the expected remaining sandbox keys are still queued properly (regression test) var actualRemaining []string namespacedWarmPoolName := queue.GetNamespacedWarmPoolName("default", "test-pool") for { - key, ok := warmSandboxQueue.Get(namespacedWarmPoolName) + key, ok := warmSandboxQueue.GetWithStrategy(namespacedWarmPoolName, + func(keys []queue.SandboxKey) (queue.SandboxKey, bool) { return keys[0], true }) if !ok { break } @@ -4456,7 +4325,7 @@ func TestCreateSandboxClaimVolumeClaimTemplatesSuccess(t *testing.T) { }, policy: extensionsv1beta1.VolumeClaimTemplatesPolicyAllowed, expectedVCTs: []string{"data", "custom"}, - expectedStorage: "2Gi", // for custom volume + expectedStorage: "2Gi", expectColdStart: true, }, { @@ -4485,7 +4354,6 @@ func TestCreateSandboxClaimVolumeClaimTemplatesSuccess(t *testing.T) { }, } - // Copy of template with VCT policy set templateCopy := template.DeepCopy() templateCopy.Spec.VolumeClaimTemplatesPolicy = tc.policy @@ -4508,7 +4376,7 @@ func TestCreateSandboxClaimVolumeClaimTemplatesSuccess(t *testing.T) { Kind: "SandboxWarmPool", Name: "vct-warmpool", UID: "pool-uid-123", - Controller: ptr.To(true), // nolint:modernize + Controller: new(true), }}, Spec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{ @@ -4551,14 +4419,12 @@ func TestCreateSandboxClaimVolumeClaimTemplatesSuccess(t *testing.T) { require.NotNil(t, controllerRef) require.Equal(t, claim.UID, controllerRef.UID) - // Verify claim's AssignedSandboxName annotation var updatedClaim extensionsv1beta1.SandboxClaim require.NoError(t, fakeClient.Get(t.Context(), req.NamespacedName, &updatedClaim)) require.Equal(t, tc.expectedAdoptedSandbox, updatedClaim.Annotations[extensionsv1beta1.AssignedSandboxNameAnnotation]) return } - // Verify newly created cold-started sandbox with propagated/merged VolumeClaimTemplates sandbox := &sandboxv1beta1.Sandbox{} err = fakeClient.Get(t.Context(), types.NamespacedName{Name: claimName, Namespace: "default"}, sandbox) require.NoError(t, err) @@ -4715,7 +4581,6 @@ func TestCreateSandboxClaimVolumeClaimTemplatesErrors(t *testing.T) { _, err := reconciler.Reconcile(t.Context(), req) require.NoError(t, err) - // Verify claim condition reflects the error status updatedClaim := &extensionsv1beta1.SandboxClaim{} err = fakeClient.Get(t.Context(), req.NamespacedName, updatedClaim) require.NoError(t, err) @@ -4910,8 +4775,6 @@ func TestStageAnnotationsElidesItsWriteAfterAnotherClaimWrite(t *testing.T) { flush := r.stageAnnotations(t.Context(), live) require.NotEmpty(t, live.Annotations[asmetrics.ObservabilityAnnotation], "annotation must be staged in memory") - // Stand in for adoption's full-object Update: it persists the staged - // annotations and advances the resourceVersion. require.NoError(t, fakeClient.Update(t.Context(), live)) require.NoError(t, flush()) require.Zero(t, patches, "flush wrote again after another claim write already carried the annotations") @@ -4941,8 +4804,7 @@ func TestStageAnnotationsWritesWhenNothingElseTouchedTheClaim(t *testing.T) { func TestExpiredClaimStillPersistsItsObservabilityAnnotation(t *testing.T) { scheme := newScheme(t) - // Already past its deadline, and no expired condition yet: the pass takes the - // status-update-and-return branch, which happens before the normal flush point. + pastTime := metav1.NewTime(time.Now().Add(-time.Hour)) claim := &extensionsv1beta1.SandboxClaim{ Name: "c", Namespace: "default", UID: "c-uid", @@ -4976,10 +4838,6 @@ func TestExpiredClaimStillPersistsItsObservabilityAnnotation(t *testing.T) { } func TestStagedAnnotationsSurviveAPartialClaimPatch(t *testing.T) { - // A partial patch (stale-reference clearing) advances the resourceVersion but - // its MergeFrom base was taken after staging, so its body carries only its own - // change. Treating any RV bump as "already written" would drop the - // observability annotation for this pass. scheme := newScheme(t) claim := &extensionsv1beta1.SandboxClaim{ Name: "c", Namespace: "default", UID: "c-uid", @@ -5004,8 +4862,6 @@ func TestStagedAnnotationsSurviveAPartialClaimPatch(t *testing.T) { } func TestFlushDoesNotResurrectAClearedAnnotation(t *testing.T) { - // The writer replays what it staged. Replaying the whole annotation map would - // undo a stale assigned-sandbox reference another step just cleared. scheme := newScheme(t) claim := &extensionsv1beta1.SandboxClaim{ Name: "c", Namespace: "default", UID: "c-uid", @@ -5031,6 +4887,96 @@ func TestFlushDoesNotResurrectAClearedAnnotation(t *testing.T) { "the staged observability annotation still has to land") } +func TestClaimSandboxPredicatePassesAReasonOnlyReadyChange(t *testing.T) { + ready := func(status metav1.ConditionStatus, reason, message string) *sandboxv1beta1.Sandbox { + return &sandboxv1beta1.Sandbox{ + Name: "sb", Namespace: "default", + Status: sandboxv1beta1.SandboxStatus{Conditions: []metav1.Condition{{ + Type: string(sandboxv1beta1.SandboxConditionReady), + Status: status, + Reason: reason, + Message: message, + }}}, + } + } + pass := claimSandboxChangePredicate().Update + + cases := []struct { + name string + old, new *sandboxv1beta1.Sandbox + want bool + }{ + { + name: "reason and message change under an unchanged False", + old: ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonDependenciesNotReady, "Pod exists with phase: Pending"), + new: ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonPodFailed, "Pod failed"), + want: true, + }, + { + name: "status flips", + old: ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonDependenciesNotReady, "Pod is Running but not Ready"), + new: ready(metav1.ConditionTrue, sandboxv1beta1.SandboxReasonDependenciesReady, "Pod is Ready"), + want: true, + }, + { + name: "identical condition", + old: ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonDependenciesNotReady, "Pod does not exist"), + new: ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonDependenciesNotReady, "Pod does not exist"), + want: false, + }, + { + name: "observed generation only", + old: ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonDependenciesNotReady, "Pod does not exist"), + new: withObservedGeneration(ready(metav1.ConditionFalse, sandboxv1beta1.SandboxReasonDependenciesNotReady, "Pod does not exist"), 2), + want: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, pass(event.UpdateEvent{ObjectOld: tc.old, ObjectNew: tc.new})) + }) + } +} + +func TestPickNodeSpreadDrainsTheFullestNodeAndBreaksTiesByArrival(t *testing.T) { + key := func(name, node string) queue.SandboxKey { + return queue.SandboxKey{Namespace: "default", Name: name, NodeName: node} + } + cases := []struct { + name string + keys []queue.SandboxKey + want string + }{ + { + name: "fullest node wins", + keys: []queue.SandboxKey{key("a", "node-1"), key("b", "node-2"), key("c", "node-2")}, + want: "b", + }, + { + name: "equal counts break by arrival", + keys: []queue.SandboxKey{key("a", "node-1"), key("b", "node-2"), key("c", "node-2"), key("d", "node-1")}, + want: "a", + }, + { + name: "unscheduled keys are skipped while a node has capacity", + keys: []queue.SandboxKey{key("a", ""), key("b", "node-1")}, + want: "b", + }, + { + name: "all unscheduled falls back to arrival order", + keys: []queue.SandboxKey{key("a", ""), key("b", "")}, + want: "a", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := pickNodeSpread(tc.keys) + require.True(t, ok) + require.Equal(t, tc.want, got.Name) + }) + } +} + func newScheme(t testing.TB) *runtime.Scheme { scheme := runtime.NewScheme() if err := sandboxv1beta1.AddToScheme(scheme); err != nil { @@ -5078,7 +5024,6 @@ func (c *conflictClient) Patch(ctx context.Context, obj client.Object, patch cli return c.Client.Patch(ctx, obj, patch, opts...) } -// countObservedTimesEntries returns the number of live entries in the observedTimes map. func countObservedTimesEntries(r *SandboxClaimReconciler) int { count := 0 r.observedTimes.inner.Range(func(_, _ any) bool { count++; return true }) @@ -5106,3 +5051,8 @@ func (m *mockTracer) IsRecording(_ context.Context) bool { } func (m *mockTracer) AddEvent(_ context.Context, _ string, _ map[string]string) {} + +func withObservedGeneration(sb *sandboxv1beta1.Sandbox, gen int64) *sandboxv1beta1.Sandbox { + sb.Status.Conditions[0].ObservedGeneration = gen + return sb +} diff --git a/extensions/controllers/sandboxclaim_pod_exclusivity_test.go b/extensions/controllers/sandboxclaim_pod_exclusivity_test.go index bf01f16..f3969a2 100644 --- a/extensions/controllers/sandboxclaim_pod_exclusivity_test.go +++ b/extensions/controllers/sandboxclaim_pod_exclusivity_test.go @@ -32,14 +32,11 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extensionsv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" - "github.com/cocoonstack/sandbox-operator/extensions/controllers/queue" "github.com/cocoonstack/sandbox-operator/internal/hash" asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/internal/queue" ) -// TestWarmPoolPodExclusivity is a regression test for the 1:1 sandbox-to-pod -// invariant (#127). When more claims exist than warm pool sandboxes, each -// sandbox must be adopted by at most one claim. func TestWarmPoolPodExclusivity(t *testing.T) { scheme := newScheme(t) ctx := t.Context() @@ -93,7 +90,6 @@ func TestWarmPoolPodExclusivity(t *testing.T) { } } - // 2 warm pool sandboxes, 3 claims — at least 1 claim must cold-start poolSb0 := createPoolSandbox("pool-sb-0") poolSb1 := createPoolSandbox("pool-sb-1") @@ -134,7 +130,6 @@ func TestWarmPoolPodExclusivity(t *testing.T) { Tracer: asmetrics.NewNoOp(), MaxConcurrentReconciles: 1, } - // Reconcile all 3 claims sequentially for _, cl := range claims { _, err := reconciler.Reconcile(ctx, reconcile.Request{ Name: cl.Name, Namespace: "default", @@ -142,11 +137,10 @@ func TestWarmPoolPodExclusivity(t *testing.T) { require.NoError(t, err, "reconcile %s", cl.Name) } - // Collect all sandboxes and build sandbox → []owning claims var allSandboxes sandboxv1beta1.SandboxList require.NoError(t, fc.List(ctx, &allSandboxes, client.InNamespace("default"))) - sandboxToOwners := make(map[string][]string) // sandbox name → [claim names] + sandboxToOwners := make(map[string][]string) for _, sb := range allSandboxes.Items { ref := metav1.GetControllerOf(&sb) if ref != nil && ref.Kind == "SandboxClaim" { @@ -154,7 +148,6 @@ func TestWarmPoolPodExclusivity(t *testing.T) { } } - // Each sandbox must be owned by at most one claim warmPoolNames := map[string]bool{"pool-sb-0": true, "pool-sb-1": true} warmAdoptions := 0 for sbName, owners := range sandboxToOwners { @@ -166,8 +159,7 @@ func TestWarmPoolPodExclusivity(t *testing.T) { } } - // Each claim must own exactly one sandbox - claimToSandbox := make(map[string][]string) // claim name → [sandbox names] + claimToSandbox := make(map[string][]string) for sbName, owners := range sandboxToOwners { for _, owner := range owners { claimToSandbox[owner] = append(claimToSandbox[owner], sbName) diff --git a/extensions/controllers/sandboxtemplate_controller.go b/extensions/controllers/sandboxtemplate_controller.go index a8911af..9cf4ec0 100644 --- a/extensions/controllers/sandboxtemplate_controller.go +++ b/extensions/controllers/sandboxtemplate_controller.go @@ -34,6 +34,10 @@ import ( asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" ) +// defaultRouterNamespace is used when RouterNamespace is unset, matching the +// operator's default install namespace. +const defaultRouterNamespace = "sandbox-system" + // SandboxTemplateReconciler reconciles a SandboxTemplate object. type SandboxTemplateReconciler struct { client.Client @@ -46,10 +50,6 @@ type SandboxTemplateReconciler struct { RouterNamespace string } -// defaultRouterNamespace is used when RouterNamespace is unset, matching the -// operator's default install namespace. -const defaultRouterNamespace = "sandbox-system" - //+kubebuilder:rbac:groups=extensions.agents.x-k8s.io,resources=sandboxtemplates,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=extensions.agents.x-k8s.io,resources=sandboxtemplates/finalizers,verbs=get;update;patch //+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete diff --git a/extensions/controllers/sandboxtemplate_controller_test.go b/extensions/controllers/sandboxtemplate_controller_test.go index 8c361aa..ce34670 100644 --- a/extensions/controllers/sandboxtemplate_controller_test.go +++ b/extensions/controllers/sandboxtemplate_controller_test.go @@ -72,7 +72,7 @@ func TestSandboxTemplateReconcileNetworkPolicy(t *testing.T) { Spec: extensionsv1beta1.SandboxTemplateSpec{ NetworkPolicyManagement: extensionsv1beta1.NetworkPolicyManagementUnmanaged, NetworkPolicy: &extensionsv1beta1.NetworkPolicySpec{ - Egress: []networkingv1.NetworkPolicyEgressRule{{}}, // Should be ignored + Egress: []networkingv1.NetworkPolicyEgressRule{{}}, }, }, } @@ -106,7 +106,7 @@ func TestSandboxTemplateReconcileNetworkPolicy(t *testing.T) { }, }, Spec: networkingv1.NetworkPolicySpec{ - PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"old-label": "outdated"}}, // Will be overwritten + PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"old-label": "outdated"}}, }, } *outdatedNPToUpdate.OwnerReferences[0].Controller = true @@ -206,7 +206,7 @@ func TestSandboxTemplateReconcileNetworkPolicy(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - scheme := newScheme(t) // Assuming newScheme is in your other test file (it's package level) + scheme := newScheme(t) client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.existingObjects...).Build() reconciler := &SandboxTemplateReconciler{ @@ -275,7 +275,6 @@ func TestSandboxTemplateReconcile_Vulnerability(t *testing.T) { }, } - // This NetworkPolicy is NOT owned by the template unownedNP := &networkingv1.NetworkPolicy{ Name: "victim-network-policy", Namespace: "default", @@ -300,7 +299,6 @@ func TestSandboxTemplateReconcile_Vulnerability(t *testing.T) { t.Fatalf("reconcile: (%v)", err) } - // Check if unownedNP still exists var np networkingv1.NetworkPolicy err = client.Get(t.Context(), types.NamespacedName{Name: "victim-network-policy", Namespace: "default"}, &np) if err != nil { @@ -320,7 +318,6 @@ func TestSandboxTemplateReconcile_Vulnerability(t *testing.T) { }, } - // This NetworkPolicy is NOT owned by the template unownedNP := &networkingv1.NetworkPolicy{ Name: "victim-network-policy", Namespace: "default", @@ -341,12 +338,10 @@ func TestSandboxTemplateReconcile_Vulnerability(t *testing.T) { } _, err := reconciler.Reconcile(t.Context(), req) - // We expect an error here once fixed, but currently it might succeed and overwrite if err != nil { t.Logf("Reconcile returned error (expected after fix): %v", err) } - // Check if unownedNP was updated var np networkingv1.NetworkPolicy err = client.Get(t.Context(), types.NamespacedName{Name: "victim-network-policy", Namespace: "default"}, &np) if err != nil { diff --git a/extensions/controllers/sandboxwarmpool_bench_test.go b/extensions/controllers/sandboxwarmpool_bench_test.go index 4f907cb..ab9e9fc 100644 --- a/extensions/controllers/sandboxwarmpool_bench_test.go +++ b/extensions/controllers/sandboxwarmpool_bench_test.go @@ -17,8 +17,6 @@ import ( const benchPoolMembers = 2500 -// BenchmarkWarmPoolReconcileSteady measures one full steady-state Reconcile of a -// pool at the validated scale — the cost every member event used to pay. func BenchmarkWarmPoolReconcileSteady(b *testing.B) { r, pool, _ := newBenchPool(b) req := ctrl.Request{Namespace: pool.Namespace, Name: pool.Name} @@ -31,9 +29,6 @@ func BenchmarkWarmPoolReconcileSteady(b *testing.B) { } } -// BenchmarkPoolMemberDeepCopy measures deep-copying the full member list — the -// per-List cost the informer cache pays for this controller unless the read is -// declared copy-free. func BenchmarkPoolMemberDeepCopy(b *testing.B) { _, _, members := newBenchPool(b) b.ReportAllocs() @@ -44,8 +39,6 @@ func BenchmarkPoolMemberDeepCopy(b *testing.B) { } } -// newBenchPool builds a reconciler over a steady pool: desired == current == -// benchPoolMembers, every member owned, warm-labeled, hash-fresh, and Ready. func newBenchPool(b *testing.B) (*SandboxWarmPoolReconciler, *extensionsv1beta1.SandboxWarmPool, []sandboxv1beta1.Sandbox) { scheme := newScheme(b) template := &extensionsv1beta1.SandboxTemplate{ diff --git a/extensions/controllers/sandboxwarmpool_controller.go b/extensions/controllers/sandboxwarmpool_controller.go index a2aad0f..55b4787 100644 --- a/extensions/controllers/sandboxwarmpool_controller.go +++ b/extensions/controllers/sandboxwarmpool_controller.go @@ -87,7 +87,6 @@ type SandboxWarmPoolReconciler struct { func (r *SandboxWarmPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - // Fetch the SandboxWarmPool instance warmPool := &extensionsv1beta1.SandboxWarmPool{} if err := r.Get(ctx, req.NamespacedName, warmPool); err != nil { if k8serrors.IsNotFound(err) { @@ -98,22 +97,18 @@ func (r *SandboxWarmPoolReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - // Handle deletion if !warmPool.DeletionTimestamp.IsZero() { logger.Info("SandboxWarmPool is being deleted") return ctrl.Result{}, nil } - // Save old status for comparison oldStatus := warmPool.Status.DeepCopy() - // Reconcile the pool (create or delete Sandboxes as needed) requeueAfter, err := r.reconcilePool(ctx, warmPool) if err != nil { return ctrl.Result{}, err } - // Update status if it has changed if err := r.updateStatus(ctx, oldStatus, warmPool); err != nil { logger.Error(err, "Failed to update SandboxWarmPool status") return ctrl.Result{}, err @@ -160,10 +155,8 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool return 0, r.reconcilePoolStatusOnly(ctx, warmPool) } - // Compute hash of the warm pool name for the pool label poolNameHash := hash.Name(warmPool.Name) - // List all Sandbox CRs with the warm pool label sandboxList := &sandboxv1beta1.SandboxList{} labelSelector := labels.SelectorFromSet(labels.Set{ warmPoolSandboxLabel: poolNameHash, @@ -180,25 +173,23 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool return 0, err } - // Fetch template and compute hash once to avoid repeated expensive operations, template, currentSandboxBlueprintHash, tmplErr := r.fetchTemplateAndHash(ctx, warmPool) - // Delete stale pods, filter pods by ownership and adopt orphans activeSandboxes, allErrors := r.filterActiveSandboxes(ctx, warmPool, sandboxList.Items, template, currentSandboxBlueprintHash, tmplErr) const warmPoolReadinessGracePeriod = 5 * time.Minute now := time.Now() - var healthySandboxes []sandboxv1beta1.Sandbox + var healthySandboxes []*sandboxv1beta1.Sandbox var stuckRecheck time.Duration for _, sb := range activeSandboxes { - if !isSandboxReady(&sb) && !sb.CreationTimestamp.IsZero() { + if !isSandboxReady(sb) && !sb.CreationTimestamp.IsZero() { age := now.Sub(sb.CreationTimestamp.Time) if age > warmPoolReadinessGracePeriod { logger.Info("Deleting stuck warm pool sandbox", "sandbox", sb.Name, "age", age.Round(time.Second)) - if err := r.Delete(ctx, &sb); err != nil { + if err := r.Delete(ctx, sb); err != nil { logger.Error(err, "Failed to delete stuck sandbox", "sandbox", sb.Name) allErrors = errors.Join(allErrors, err) } @@ -227,10 +218,9 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool warmPool.Status.Replicas = currentReplicas warmPool.Status.Selector = labelSelector.String() - // Calculate ready replicas by checking Sandbox Ready condition readyReplicas := int32(0) for i := range activeSandboxes { - if isSandboxReady(&activeSandboxes[i]) { + if isSandboxReady(activeSandboxes[i]) { readyReplicas++ } } @@ -264,9 +254,9 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool // Prioritize deleting unready sandboxes before ready ones, // then newest first within each group. - slices.SortFunc(activeSandboxes, func(a, b sandboxv1beta1.Sandbox) int { - aReady := isSandboxReady(&a) - bReady := isSandboxReady(&b) + slices.SortFunc(activeSandboxes, func(a, b *sandboxv1beta1.Sandbox) int { + aReady := isSandboxReady(a) + bReady := isSandboxReady(b) if aReady != bReady { if aReady { return 1 // a ready, b not ready -> b first (delete unready first) @@ -279,7 +269,7 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool toDeleteCount := min(sandboxesToDelete, int32(len(activeSandboxes))) // Parallel sandbox deletion with adaptive slow-start batching (starts with 1 and doubles on success) _, deleteErr := slowStartBatch(ctx, int(toDeleteCount), 1, func(idx int) error { - return r.deletePoolSandbox(ctx, &activeSandboxes[idx]) + return r.deletePoolSandbox(ctx, activeSandboxes[idx]) }) if deleteErr != nil { logger.Error(deleteErr, "Failed to delete pool sandboxes") @@ -336,9 +326,9 @@ func (r *SandboxWarmPoolReconciler) adoptSandbox(ctx context.Context, warmPool * } // filterActiveSandboxes filters the list of sandboxes, deleting stale ones and adopting orphans. -func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, warmPool *extensionsv1beta1.SandboxWarmPool, sandboxes []sandboxv1beta1.Sandbox, template *extensionsv1beta1.SandboxTemplate, currentSandboxBlueprintHash string, tmplErr error) ([]sandboxv1beta1.Sandbox, error) { +func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, warmPool *extensionsv1beta1.SandboxWarmPool, sandboxes []sandboxv1beta1.Sandbox, template *extensionsv1beta1.SandboxTemplate, currentSandboxBlueprintHash string, tmplErr error) ([]*sandboxv1beta1.Sandbox, error) { logger := log.FromContext(ctx) - var activeSandboxes []sandboxv1beta1.Sandbox + var activeSandboxes []*sandboxv1beta1.Sandbox var allErrors error vettedHashes := make(map[string]bool) @@ -347,7 +337,6 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w currentTemplateRefHash = SandboxTemplateRefHash(template.Name) } - // Determine the update strategy, defaulting to OnReplenish if not specified or unknown. var updateStrategyType extensionsv1beta1.SandboxWarmPoolUpdateStrategyType if warmPool.Spec.UpdateStrategy != nil { updateStrategyType = warmPool.Spec.UpdateStrategy.Type @@ -364,12 +353,13 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w updateStrategy = extensionsv1beta1.OnReplenishSandboxWarmPoolUpdateStrategyType } - for _, sb := range sandboxes { + for i := range sandboxes { + sb := &sandboxes[i] if !sb.DeletionTimestamp.IsZero() { continue } - controllerRef := metav1.GetControllerOf(&sb) + controllerRef := metav1.GetControllerOf(sb) isOrphan := controllerRef == nil isControlledByPool := controllerRef != nil && controllerRef.UID == warmPool.UID @@ -379,9 +369,9 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w } if tmplErr == nil && (updateStrategy == extensionsv1beta1.RecreateSandboxWarmPoolUpdateStrategyType || isOrphan) { - if r.isSandboxStale(ctx, &sb, template, currentTemplateRefHash, currentSandboxBlueprintHash, vettedHashes) { + if r.isSandboxStale(ctx, sb, template, currentTemplateRefHash, currentSandboxBlueprintHash, vettedHashes) { logger.Info("Deleting stale sandbox", "sandbox", sb.Name, "isOrphan", isOrphan) - if err := r.Delete(ctx, &sb); err != nil { + if err := r.Delete(ctx, sb); err != nil { logger.Error(err, "Failed to delete stale sandbox", "sandbox", sb.Name) allErrors = errors.Join(allErrors, err) } @@ -398,7 +388,7 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w allErrors = errors.Join(allErrors, err) continue } - sb = *fresh + sb = fresh } if isOrphan { @@ -409,7 +399,7 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w allErrors = errors.Join(allErrors, err) continue } - sb = *fresh + sb = fresh } activeSandboxes = append(activeSandboxes, sb) @@ -462,13 +452,11 @@ func (r *SandboxWarmPoolReconciler) buildSandboxCR( Namespace: warmPool.Namespace, Labels: sandboxLabels, Annotations: sandboxAnnotations, - // Deep-copy the entire shared blueprint Spec: sandboxv1beta1.SandboxSpec{ SandboxBlueprint: *template.Spec.SandboxBlueprint.DeepCopy(), }, } - // Propagate pool and template labels to pod template for consistency and targeting if sandbox.Spec.PodTemplate.ObjectMeta.Labels == nil { sandbox.Spec.PodTemplate.ObjectMeta.Labels = make(map[string]string) } @@ -487,7 +475,6 @@ func (r *SandboxWarmPoolReconciler) buildSandboxCR( } } - // Apply secure defaults to the sandbox pod spec ApplySandboxSecureDefaults(template, &sandbox.Spec.PodTemplate.Spec) if err := ctrl.SetControllerReference(warmPool, sandbox, r.Scheme); err != nil { @@ -525,7 +512,6 @@ func (r *SandboxWarmPoolReconciler) deletePoolSandbox(ctx context.Context, sb *s func (r *SandboxWarmPoolReconciler) updateStatus(ctx context.Context, oldStatus *extensionsv1beta1.SandboxWarmPoolStatus, warmPool *extensionsv1beta1.SandboxWarmPool) error { logger := log.FromContext(ctx) - // Check if status has changed if equality.Semantic.DeepEqual(oldStatus, &warmPool.Status) { return nil } @@ -570,45 +556,35 @@ func (r *SandboxWarmPoolReconciler) isSandboxStale( ) bool { sandboxHash := sandbox.Labels[sandboxv1beta1.SandboxTemplateHashLabel] - // If the templateRefHash doesn't match, it's stale. if sandbox.Labels[sandboxTemplateRefHash] != currentTemplateRefHash { return true } - // Check if the sandbox is unowned (orphaned). controllerRef := metav1.GetControllerOf(sandbox) isOrphan := controllerRef == nil if isOrphan { - // Always perform full semantic comparison for orphans. return !r.compareSandboxBlueprint(template, &sandbox.Spec.SandboxBlueprint) } - // If hashes match, it's fresh. if sandboxHash != "" && sandboxHash == currentSandboxBlueprintHash { return false } - // If currentSandboxBlueprintHash is empty, it means we failed to compute it. - // In this case, we should log an error and treat it as NOT stale to avoid - // mass-deleting existing sandboxes due to a marshal failure. + // A marshal failure leaves the hash empty; treating that as stale would + // mass-delete the pool. if currentSandboxBlueprintHash == "" { log.FromContext(ctx).Error(nil, "currentSandboxBlueprintHash is empty, skipping staleness check", "sandbox", sandbox.Name) return false } - // Check if we've already evaluated this specific old version. if sandboxHash != "" { if isStale, found := vettedHashes[sandboxHash]; found { return isStale } } - // Perform a semantic comparison of the sandbox blueprint. - // We normalize the pod spec by applying the same secure defaults - // used during creation to avoid false positives from controller-injected fields. isStale := !r.compareSandboxBlueprint(template, &sandbox.Spec.SandboxBlueprint) - // Save the result for the next sandbox with this same hash. if sandboxHash != "" { vettedHashes[sandboxHash] = isStale } @@ -619,13 +595,10 @@ func (r *SandboxWarmPoolReconciler) isSandboxStale( // comparePodSpecs checks if the pod spec in the sandbox is semantically equal to the template, // normalizing for fields that the controller populates by default. func (r *SandboxWarmPoolReconciler) comparePodSpecs(template *extensionsv1beta1.SandboxTemplate, actualSandboxSpec *corev1.PodSpec) bool { - // Create what the sandbox SHOULD look like if it were created from the current template. expectedSpec := template.Spec.PodTemplate.Spec.DeepCopy() ApplySandboxSecureDefaults(template, expectedSpec) - // Compare the actual sandbox spec to the expected "perfect" spec. - // Since both have now undergone the exact same defaulting logic, - // any remaining difference is a TRUE template drift. + // Both sides carry the same defaulting, so a remaining difference is drift. return equality.Semantic.DeepEqual(expectedSpec, actualSandboxSpec) } diff --git a/extensions/controllers/sandboxwarmpool_controller_test.go b/extensions/controllers/sandboxwarmpool_controller_test.go index e6677e0..9eeecef 100644 --- a/extensions/controllers/sandboxwarmpool_controller_test.go +++ b/extensions/controllers/sandboxwarmpool_controller_test.go @@ -144,7 +144,6 @@ func TestReconcilePool(t *testing.T) { _, err = r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Verify final state - count sandboxes with correct warm pool label list := &sandboxv1beta1.SandboxList{} err = r.List(ctx, list, &client.ListOptions{Namespace: poolNamespace}) require.NoError(t, err) @@ -373,11 +372,9 @@ func TestPoolLabelValueInIntegration(t *testing.T) { require.Equal(t, sandboxv1beta1.SandboxLaunchTypeWarm, sb.Labels[sandboxv1beta1.SandboxLaunchTypeLabel], "sandbox %s should have warm launch type label", sb.Name) - // Verify pod template labels are propagated into the sandbox's pod template require.Equal(t, "2.0", sb.Spec.PodTemplate.ObjectMeta.Labels["version"]) require.Equal(t, "from-podtemplate", sb.Spec.PodTemplate.ObjectMeta.Labels["pod-label"]) - // Verify pod template annotations require.Equal(t, "from-podtemplate", sb.Spec.PodTemplate.ObjectMeta.Annotations["pod-annotation"]) require.Equal(t, "true", sb.Spec.PodTemplate.ObjectMeta.Annotations[warmPoolEvictionAnnotation]) } @@ -747,12 +744,10 @@ func TestReconcilePoolGCStuckSandboxes(t *testing.T) { _, err := r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // The stuck sandbox should be deleted and replaced list := &sandboxv1beta1.SandboxList{} err = r.List(ctx, list, &client.ListOptions{Namespace: poolNamespace}) require.NoError(t, err) - // Should have: 1 healthy (kept) + 1 newly created replacement = 2 poolCount := int32(0) for _, sb := range list.Items { if sb.Labels[warmPoolSandboxLabel] == poolNameHash { @@ -777,7 +772,6 @@ func TestReconcilePoolGCStuckSandboxes(t *testing.T) { _, err := r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Both should be kept (one healthy, one still within grace period) list := &sandboxv1beta1.SandboxList{} err = r.List(ctx, list, &client.ListOptions{Namespace: poolNamespace}) require.NoError(t, err) @@ -823,7 +817,6 @@ func TestReconcilePool_TemplateUpdateRollout(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Create initial SandboxTemplate template := &extensionsv1beta1.SandboxTemplate{ APIVersion: extensionsv1beta1.GroupVersion.String(), Kind: "SandboxTemplate", @@ -867,15 +860,12 @@ func TestReconcilePool_TemplateUpdateRollout(t *testing.T) { ctx := t.Context() - // Initial reconciliation to create the sandboxes _, err := r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Get initial hash label template, initialHash, err := r.fetchTemplateAndHash(ctx, warmPool) require.NoError(t, err) - // Verify sandboxes exist with initial image and hash sandboxes := &sandboxv1beta1.SandboxList{} err = r.List(ctx, sandboxes, client.InNamespace(poolNamespace)) require.NoError(t, err) @@ -885,51 +875,44 @@ func TestReconcilePool_TemplateUpdateRollout(t *testing.T) { require.Equal(t, initialHash, sb.Labels[sandboxv1beta1.SandboxTemplateHashLabel], "Sandbox should have initial sandbox blueprint hash label") } - // Update the SandboxTemplate content updatedTemplate := template.DeepCopy() updatedTemplate.Spec.PodTemplate.Spec.Containers[0].Image = "image-v2" err = r.Update(ctx, updatedTemplate) require.NoError(t, err) - // Get new expected hash label _, updatedHash, err := r.fetchTemplateAndHash(ctx, warmPool) require.NoError(t, err) require.NotEqual(t, initialHash, updatedHash, "Hashes should differ after template update") - // Reconcile again to trigger rollout (or lack thereof) _, err = r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Verify state after update err = r.List(ctx, sandboxes, client.InNamespace(poolNamespace)) require.NoError(t, err) require.Len(t, sandboxes.Items, int(replicas)) if tc.expectedUpdatedImage { - // For Recreate strategy, all should be updated + for _, sb := range sandboxes.Items { require.Equal(t, "image-v2", sb.Spec.PodTemplate.Spec.Containers[0].Image, "Sandbox should have updated image") require.Equal(t, updatedHash, sb.Labels[sandboxv1beta1.SandboxTemplateHashLabel], "Sandbox should have updated sandbox blueprint hash label") } t.Log("Verified: All sandboxes updated immediately with Recreate strategy") } else { - // For OnReplenish (default), all should still be v1 + for _, sb := range sandboxes.Items { require.Equal(t, "image-v1", sb.Spec.PodTemplate.Spec.Containers[0].Image, "Sandbox should retain original image") require.Equal(t, initialHash, sb.Labels[sandboxv1beta1.SandboxTemplateHashLabel], "Sandbox should retain original sandbox blueprint hash label") } t.Log("Verified: Sandboxes retained original image after update with OnReplenish strategy") - // Now manually delete one sandbox to test replenishment sbToDelete := &sandboxes.Items[0] err = r.Delete(ctx, sbToDelete) require.NoError(t, err) - // Reconcile to trigger replenishment _, err = r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Verify that we have 2 sandboxes: one old (v1) and one new (v2) err = r.List(ctx, sandboxes, client.InNamespace(poolNamespace)) require.NoError(t, err) require.Len(t, sandboxes.Items, int(replicas)) @@ -960,7 +943,6 @@ func TestReconcilePool_TemplateRefUpdate_SameSpec(t *testing.T) { templateName2 := "test-template-2" replicas := int32(2) - // Create initial SandboxTemplate template1 := &extensionsv1beta1.SandboxTemplate{ APIVersion: extensionsv1beta1.GroupVersion.String(), Kind: "SandboxTemplate", @@ -1004,7 +986,6 @@ func TestReconcilePool_TemplateRefUpdate_SameSpec(t *testing.T) { ctx := t.Context() - // Initial reconcile _, err := r.reconcilePool(ctx, warmPool) require.NoError(t, err) @@ -1018,7 +999,6 @@ func TestReconcilePool_TemplateRefUpdate_SameSpec(t *testing.T) { initialSandboxNames[sb.Name] = true } - // Create new SandboxTemplate with SAME spec template2 := &extensionsv1beta1.SandboxTemplate{ APIVersion: extensionsv1beta1.GroupVersion.String(), Kind: "SandboxTemplate", @@ -1029,25 +1009,22 @@ func TestReconcilePool_TemplateRefUpdate_SameSpec(t *testing.T) { err = r.Create(ctx, template2) require.NoError(t, err) - // Update WarmPool to point to template2 warmPool.Spec.TemplateRef.Name = templateName2 err = r.Update(ctx, warmPool) require.NoError(t, err) - // Reconcile again to trigger rollout _, err = r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Verify state after update err = r.List(ctx, sandboxes, client.InNamespace(poolNamespace)) require.NoError(t, err) require.Len(t, sandboxes.Items, int(replicas)) for _, sb := range sandboxes.Items { - // Sandboxes should be recreated (new names) because TemplateRef changed + require.False(t, initialSandboxNames[sb.Name], "Sandbox should have been recreated with new name") require.Equal(t, hash.Name(templateName2), sb.Labels[sandboxTemplateRefHash], "Sandbox should have updated template ref hash label") - // The pod spec is identical, so the image remains image-v1 + require.Equal(t, "image-v1", sb.Spec.PodTemplate.Spec.Containers[0].Image, "Sandbox should retain original image since spec is identical") } } @@ -1103,7 +1080,7 @@ func TestComparePodSpecsNormalization(t *testing.T) { templateSpec corev1.PodSpec actualSpec corev1.PodSpec secureByDef bool - expectedResult bool // true if they should be considered equal + expectedResult bool }{ { name: "Identical specs should match", @@ -1182,10 +1159,8 @@ func TestComparePodSpecsNormalization(t *testing.T) { template.Spec.NetworkPolicyManagement = extensionsv1beta1.NetworkPolicyManagementUnmanaged } - // We need to apply the SAME defaults to the 'actual' spec in the test - // if we want to simulate a sandbox that was created with those defaults. actualSpecCopy := tt.actualSpec.DeepCopy() - // Only apply if it's NOT a drift test case where we WANT them to be different + if tt.expectedResult { ApplySandboxSecureDefaults(template, actualSpecCopy) } @@ -1207,7 +1182,6 @@ func TestReconcilePool_TemplateUpdate_DNSPolicy(t *testing.T) { ctx := t.Context() scheme := newTestScheme() - // Create initial SandboxTemplate with default DNS template := &extensionsv1beta1.SandboxTemplate{ Name: templateName, Namespace: poolNamespace, @@ -1244,11 +1218,9 @@ func TestReconcilePool_TemplateUpdate_DNSPolicy(t *testing.T) { MaxBatchSize: sandboxCreateDeleteMaxBatchSize, } - // Initial reconcile to create sandboxes _, err := r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Verify initial state sandboxes := &sandboxv1beta1.SandboxList{} err = r.List(ctx, sandboxes, client.InNamespace(poolNamespace)) require.NoError(t, err) @@ -1257,17 +1229,14 @@ func TestReconcilePool_TemplateUpdate_DNSPolicy(t *testing.T) { require.Equal(t, corev1.DNSDefault, sb.Spec.PodTemplate.Spec.DNSPolicy) } - // Update SandboxTemplate to change DNSPolicy updatedTemplate := template.DeepCopy() updatedTemplate.Spec.PodTemplate.Spec.DNSPolicy = corev1.DNSClusterFirst err = r.Update(ctx, updatedTemplate) require.NoError(t, err) - // Reconcile again, should trigger rollout (deletion and recreation) _, err = r.reconcilePool(ctx, warmPool) require.NoError(t, err) - // Verify that sandboxes now have the updated DNSPolicy err = r.List(ctx, sandboxes, client.InNamespace(poolNamespace)) require.NoError(t, err) require.Len(t, sandboxes.Items, int(replicas)) @@ -1304,8 +1273,6 @@ func TestIsSandboxStale_OrphanedSandboxVetting(t *testing.T) { r := &SandboxWarmPoolReconciler{Scheme: scheme} vettedHashes := make(map[string]bool) - // Case 1: Orphaned sandbox with matching hash label but modified PodSpec (Spoofed). - // Should be detected as stale because unowned sandboxes must undergo full vetting. spoofedSpec := template.Spec.PodTemplate.Spec.DeepCopy() spoofedSpec.Containers[0].Image = "malicious-image" @@ -1323,8 +1290,6 @@ func TestIsSandboxStale_OrphanedSandboxVetting(t *testing.T) { isStaleSpoofed := r.isSandboxStale(ctx, spoofedOrphan, template, SandboxTemplateRefHash(template.Name), currentSandboxBlueprintHash, vettedHashes) require.True(t, isStaleSpoofed, "Orphaned sandbox with spoofed hash but modified PodSpec should be stale") - // Case 2: Orphaned sandbox with matching hash label and genuine/fully vetted PodSpec. - // Should be evaluated as fresh (not stale) after passing full semantic comparison. genuineSpec := template.Spec.PodTemplate.Spec.DeepCopy() ApplySandboxSecureDefaults(template, genuineSpec) @@ -1374,17 +1339,17 @@ func TestSlowStartBatch(t *testing.T) { count: 14, initialBatchSize: 1, failAtIndices: new(5), - expectedSuccess: 6, // index 0, 1, 2, 3, 4, and 6 succeeds, 5 fails - 6 successful calls + expectedSuccess: 6, expectError: true, - expectedCallCount: 7, // 1 + 2 + 4 = 7 calls in total. + expectedCallCount: 7, expectedErrMsgs: []string{"injected error at idx 5"}, }, { name: "context canceled in middle of batch", count: 14, initialBatchSize: 1, - cancelContextAtIdx: new(2), // cancels during batch 2 (indices 1, 2) - expectedSuccess: 3, // indices 0, 1, 2 complete successfully before cancellation aborts batch 3 + cancelContextAtIdx: new(2), + expectedSuccess: 3, expectError: true, expectedCallCount: 3, expectedErrMsgs: []string{"context canceled"}, @@ -1706,7 +1671,6 @@ func TestReconcilePool_TemplateUpdateRecreate(t *testing.T) { ctx := t.Context() - // Initial reconcile _, err := r.reconcilePool(ctx, warmPool) require.NoError(t, err) @@ -1715,14 +1679,11 @@ func TestReconcilePool_TemplateUpdateRecreate(t *testing.T) { require.NoError(t, err) require.Len(t, sandboxes.Items, int(replicas), "expected warm sandbox after initial reconcile") - // Capture initial sandboxblueprint hash _, initialHash, err := r.fetchTemplateAndHash(ctx, warmPool) require.NoError(t, err) - // Capture initial sandbox names to verify recreation later initialName := sandboxes.Items[0].Name - // Apply the template drift if tt.updateFn != nil { updatedTemplate := template.DeepCopy() tt.updateFn(updatedTemplate) @@ -1730,14 +1691,12 @@ func TestReconcilePool_TemplateUpdateRecreate(t *testing.T) { require.NoError(t, err) } - // Capture updated sandbox blueprint hash after template update _, updatedHash, err := r.fetchTemplateAndHash(ctx, warmPool) require.NoError(t, err) if tt.expectRecreation { require.NotEqual(t, initialHash, updatedHash, "sandbox blueprint hash should change after template update") } - // Recreate strategy should delete stale sandbox and create a fresh one _, err = r.reconcilePool(ctx, warmPool) require.NoError(t, err) @@ -2026,10 +1985,6 @@ func TestCompareSandboxBlueprint(t *testing.T) { } } -// TestSandboxBlueprintFieldsAreCompared verifies that compareSandboxBlueprint() -// accounts for all fields in the SandboxBlueprint struct. A field missing from the -// comparison logic is not tracked for drift, so a warm sandbox will not be detected -// as stale when that field changes. func TestSandboxBlueprintFieldsAreCompared(t *testing.T) { expectedFields := []string{"PodTemplate", "VolumeClaimTemplates", "Service"} @@ -2049,8 +2004,6 @@ func TestSandboxBlueprintFieldsAreCompared(t *testing.T) { } func TestNewPoolSandboxesCarryOnlyTheRenamedHashLabel(t *testing.T) { - // The write path for the pre-rename label is retired; only objects created - // before the rename may still carry it, and they drain as the pool recycles. r := &SandboxWarmPoolReconciler{Scheme: newScheme(t)} warmPool := &extensionsv1beta1.SandboxWarmPool{ Name: "p", Namespace: "default", @@ -2073,7 +2026,6 @@ func TestNewPoolSandboxesCarryOnlyTheRenamedHashLabel(t *testing.T) { } } -// Create a test scheme with extensions types registered. func newTestScheme() *runtime.Scheme { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -2101,7 +2053,7 @@ func createPoolSandbox(poolName, namespace, poolNameHash string, template *exten templateRefHash = hash.Name(template.Name) podSpec = *template.Spec.PodTemplate.Spec.DeepCopy() ApplySandboxSecureDefaults(template, &podSpec) - // If template has a version label, we could use it as part of the hash placeholder + if v, ok := template.Spec.PodTemplate.ObjectMeta.Labels["version"]; ok { podTemplateHash = "pod-hash-" + v sandboxBlueprintHash = "blueprint-hash-" + v @@ -2113,7 +2065,7 @@ func createPoolSandbox(poolName, namespace, poolNameHash string, template *exten sandboxBlueprintHash = hash.Name(string(sandboxBlueprintJSON)) } } else { - // Fallback for tests that don't provide a template + podSpec = corev1.PodSpec{ Containers: []corev1.Container{ { diff --git a/extensions/controllers/utils.go b/extensions/controllers/utils.go index 66553f1..a871b32 100644 --- a/extensions/controllers/utils.go +++ b/extensions/controllers/utils.go @@ -23,27 +23,20 @@ import ( // ApplySandboxSecureDefaults applies the controller's "Secure by Default" logic to a PodSpec. func ApplySandboxSecureDefaults(template *extensionsv1beta1.SandboxTemplate, spec *corev1.PodSpec) { - // Enforce a secure-by-default policy by disabling the automatic mounting - // of the service account token, adhering to security best practices for - // sandboxed environments. if spec.AutomountServiceAccountToken == nil { - automount := false - spec.AutomountServiceAccountToken = &automount + spec.AutomountServiceAccountToken = new(false) } - // Determine if we are in "Secure By Default" mode management := template.Spec.NetworkPolicyManagement isManaged := management == "" || management == extensionsv1beta1.NetworkPolicyManagementManaged isSecureByDefault := isManaged && template.Spec.NetworkPolicy == nil - // To prevent internal DNS enumeration while still allowing public domain resolution, - // we explicitly override the Pod's DNS config to use external public resolvers. - // We only inject this if using the strict "Secure by Default" policy. If the user - // provides custom rules or is Unmanaged, we leave DNS alone for air-gapped/proxy compatibility. + // Public resolvers block internal DNS enumeration; custom rules or Unmanaged + // keep the cluster's own DNS, which air-gapped and proxied clusters need. if isSecureByDefault && spec.DNSPolicy == "" { spec.DNSPolicy = corev1.DNSNone spec.DNSConfig = &corev1.PodDNSConfig{ - Nameservers: []string{"8.8.8.8", "1.1.1.1"}, // Google & Cloudflare public DNS + Nameservers: []string{"8.8.8.8", "1.1.1.1"}, } } } diff --git a/extensions/controllers/utils_test.go b/extensions/controllers/utils_test.go index 664d184..f365e88 100644 --- a/extensions/controllers/utils_test.go +++ b/extensions/controllers/utils_test.go @@ -56,7 +56,6 @@ func TestSandboxTemplateRefHash(t *testing.T) { }) } - // Check that different inputs produce different hashes for descA, resultA := range results { for descB, resultB := range results { if descA == descB { diff --git a/go.mod b/go.mod index eb8a9a3..e4e9647 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cocoonstack/sandbox-operator go 1.27.0 require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -82,7 +82,7 @@ require ( gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/grpc v1.82.1 // indirect + google.golang.org/grpc v1.83.1 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 50570eb..de892d1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= @@ -303,8 +303,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= -google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/hack/crd-ref-docs.yaml b/hack/crd-ref-docs.yaml new file mode 100644 index 0000000..4923341 --- /dev/null +++ b/hack/crd-ref-docs.yaml @@ -0,0 +1,5 @@ +processor: + ignoreTypes: [] + ignoreFields: [] +render: + kubernetesVersion: "1.34" diff --git a/helm/crds/extensions.agents.x-k8s.io_nodeinventories.yaml b/helm/crds/extensions.agents.x-k8s.io_nodeinventories.yaml index 772f6c5..f2ae849 100644 --- a/helm/crds/extensions.agents.x-k8s.io_nodeinventories.yaml +++ b/helm/crds/extensions.agents.x-k8s.io_nodeinventories.yaml @@ -45,6 +45,8 @@ spec: type: string phase: type: string + template: + type: string required: - name - phase diff --git a/internal/hash/hash.go b/internal/hash/hash.go index d5a1ebf..02abf97 100644 --- a/internal/hash/hash.go +++ b/internal/hash/hash.go @@ -8,11 +8,10 @@ import ( // Name returns the FNV-1a hash of s as an 8-character hexadecimal string. func Name(s string) string { - return fmt.Sprintf("%08x", Numeric(s)) + return fmt.Sprintf("%08x", numeric(s)) } -// Numeric returns the 32-bit FNV-1a hash of s. -func Numeric(s string) uint32 { +func numeric(s string) uint32 { h := fnv.New32a() _, _ = h.Write([]byte(s)) return h.Sum32() diff --git a/internal/lifecycle/expiry.go b/internal/lifecycle/expiry.go index 023fe33..e8ab257 100644 --- a/internal/lifecycle/expiry.go +++ b/internal/lifecycle/expiry.go @@ -30,13 +30,25 @@ func FinishedCondition(conditions []metav1.Condition, conditionType string) *met return condition } -// NeedsCleanup reports whether ttl-after-finished cleanup applies. -func NeedsCleanup(ttlSecondsAfterFinished *int32, finishedCondition *metav1.Condition) bool { +// TimeLeft reports whether the resource has expired and, if not, how long remains. +func TimeLeft(now time.Time, shutdownTime *metav1.Time, ttlSecondsAfterFinished *int32, finishedCondition *metav1.Condition) (bool, time.Duration) { + expireAt := expireAtFor(shutdownTime, ttlSecondsAfterFinished, finishedCondition) + if expireAt == nil { + return false, 0 + } + if !now.Before(*expireAt) { + return true, 0 + } + return false, expireAt.Sub(now) +} + +// needsCleanup reports whether ttl-after-finished cleanup applies. +func needsCleanup(ttlSecondsAfterFinished *int32, finishedCondition *metav1.Condition) bool { return ttlSecondsAfterFinished != nil && finishedCondition != nil } -// FinishedTime returns the finish timestamp encoded in the terminal condition. -func FinishedTime(finishedCondition *metav1.Condition) *time.Time { +// finishedTime returns the finish timestamp encoded in the terminal condition. +func finishedTime(finishedCondition *metav1.Condition) *time.Time { if finishedCondition == nil || finishedCondition.LastTransitionTime.IsZero() { return nil } @@ -44,19 +56,19 @@ func FinishedTime(finishedCondition *metav1.Condition) *time.Time { return &finishedAt } -// ExpireAt returns the earliest configured expiry time. -func ExpireAt(shutdownTime *metav1.Time, ttlSecondsAfterFinished *int32, finishedCondition *metav1.Condition) *time.Time { +// expireAtFor returns the earliest configured expiry time. +func expireAtFor(shutdownTime *metav1.Time, ttlSecondsAfterFinished *int32, finishedCondition *metav1.Condition) *time.Time { var expireAt *time.Time if shutdownTime != nil { shutdownAt := shutdownTime.Time expireAt = &shutdownAt } - if !NeedsCleanup(ttlSecondsAfterFinished, finishedCondition) { + if !needsCleanup(ttlSecondsAfterFinished, finishedCondition) { return expireAt } - finishedAt := FinishedTime(finishedCondition) + finishedAt := finishedTime(finishedCondition) if finishedAt == nil { return expireAt } @@ -68,15 +80,3 @@ func ExpireAt(shutdownTime *metav1.Time, ttlSecondsAfterFinished *int32, finishe return expireAt } - -// TimeLeft reports whether the resource has expired and, if not, how long remains. -func TimeLeft(now time.Time, shutdownTime *metav1.Time, ttlSecondsAfterFinished *int32, finishedCondition *metav1.Condition) (bool, time.Duration) { - expireAt := ExpireAt(shutdownTime, ttlSecondsAfterFinished, finishedCondition) - if expireAt == nil { - return false, 0 - } - if !now.Before(*expireAt) { - return true, 0 - } - return false, expireAt.Sub(now) -} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 2c20059..fb81c5f 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -118,8 +118,6 @@ func TestBuildInfo(t *testing.T) { } func TestStartSpanEndFuncEndsSpan(t *testing.T) { - // StartSpan returns an end func; if the caller never invokes it, span.End is never called and - // the span is never exported, a span resource leak. This mini test just proves the func closes the span. exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) t.Cleanup(func() { _ = tp.Shutdown(t.Context()) }) diff --git a/internal/metrics/sandbox_collector.go b/internal/metrics/sandbox_collector.go index 876342f..b15b5c3 100644 --- a/internal/metrics/sandbox_collector.go +++ b/internal/metrics/sandbox_collector.go @@ -17,6 +17,7 @@ package metrics import ( "context" + "errors" "time" "github.com/go-logr/logr" @@ -48,34 +49,6 @@ type AgentSandboxesMetricKey struct { CreatedBy string } -// NewAgentSandboxesConstMetric creates a new Prometheus ConstMetric for the agent_sandboxes gauge. -func NewAgentSandboxesConstMetric(count int, key AgentSandboxesMetricKey) prometheus.Metric { - return prometheus.MustNewConstMetric( - AgentSandboxesDesc, - prometheus.GaugeValue, - float64(count), - key.Namespace, - key.ReadyCondition, - key.Expired, - key.LaunchType, - key.Template, - key.OwnedBy, - key.CreatedBy, - ) -} - -// RegisterSandboxCollector registers the custom Prometheus collector for sandbox counts. -func RegisterSandboxCollector(ctx context.Context, c client.Client, logger logr.Logger) { - collector := NewSandboxCollector(ctx, c, logger) - if err := metrics.Registry.Register(collector); err != nil { - if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { - logger.Error(err, "Failed to register SandboxCollector") - } else { - logger.Info("SandboxCollector already registered, ignoring") - } - } -} - // SandboxCollector is a custom Prometheus collector that dynamically fetches sandbox counts. type SandboxCollector struct { // baseCtx is the process lifetime context; Collect derives its scrape @@ -96,28 +69,25 @@ func NewSandboxCollector(ctx context.Context, c client.Client, logger logr.Logge } } -// Describe sends the metric descriptor to the channel. func (c *SandboxCollector) Describe(ch chan<- *prometheus.Desc) { ch <- c.agentSandboxesDesc } -// Collect fetches sandboxes, calculates labels, and sends metrics to the channel. -// UnsafeDisableDeepCopy avoids O(N) deep-copy overhead on every scrape; safe here because -// Collect only reads fields for label aggregation and never mutates or retains the objects. -// A GaugeVec updated in the Reconcile loop would be more performant (O(1) per scrape), -// but this is a known trade-off to keep the Reconcile loop simpler. func (c *SandboxCollector) Collect(ch chan<- prometheus.Metric) { var sandboxList sandboxv1beta1.SandboxList ctx, cancel := context.WithTimeout(c.baseCtx, metricsCollectTimeout) defer cancel() + // Copy-free cache read: the loop below only reads label inputs, and neither + // mutates nor retains an item. if err := c.client.List(ctx, &sandboxList, client.UnsafeDisableDeepCopy); err != nil { c.logger.Error(err, "Failed to list sandboxes for metrics collection") return } counts := make(map[AgentSandboxesMetricKey]int) - for _, sandbox := range sandboxList.Items { + for i := range sandboxList.Items { + sandbox := &sandboxList.Items[i] readyConditionStr := "false" expiredStr := "false" readyCond := meta.FindStatusCondition(sandbox.Status.Conditions, string(sandboxv1beta1.SandboxConditionReady)) @@ -136,15 +106,13 @@ func (c *SandboxCollector) Collect(ch chan<- prometheus.Metric) { } sandboxTemplateStr := "unknown" - // If a user manually creates a Sandbox without a SandboxClaim, it won't have the - // SandboxTemplateRefAnnotation. The collector correctly handles this by defaulting to "unknown". if template, ok := sandbox.Annotations[sandboxv1beta1.SandboxTemplateRefAnnotation]; ok && template != "" { sandboxTemplateStr = template } apiVersion := extensionsv1beta1.GroupVersion.String() ownedByStr := "None" - if controllerRef := metav1.GetControllerOf(&sandbox); controllerRef != nil { + if controllerRef := metav1.GetControllerOf(sandbox); controllerRef != nil { if controllerRef.APIVersion == apiVersion { switch controllerRef.Kind { case kindSandboxClaim: @@ -176,3 +144,31 @@ func (c *SandboxCollector) Collect(ch chan<- prometheus.Metric) { ch <- NewAgentSandboxesConstMetric(count, key) } } + +// NewAgentSandboxesConstMetric creates a new Prometheus ConstMetric for the agent_sandboxes gauge. +func NewAgentSandboxesConstMetric(count int, key AgentSandboxesMetricKey) prometheus.Metric { + return prometheus.MustNewConstMetric( + AgentSandboxesDesc, + prometheus.GaugeValue, + float64(count), + key.Namespace, + key.ReadyCondition, + key.Expired, + key.LaunchType, + key.Template, + key.OwnedBy, + key.CreatedBy, + ) +} + +// RegisterSandboxCollector registers the custom Prometheus collector for sandbox counts. +func RegisterSandboxCollector(ctx context.Context, c client.Client, logger logr.Logger) { + collector := NewSandboxCollector(ctx, c, logger) + if err := metrics.Registry.Register(collector); err != nil { + if _, ok := errors.AsType[prometheus.AlreadyRegisteredError](err); ok { + logger.Info("SandboxCollector already registered, ignoring") + return + } + logger.Error(err, "Failed to register SandboxCollector") + } +} diff --git a/internal/metrics/sandbox_collector_test.go b/internal/metrics/sandbox_collector_test.go index c2e6df8..2b66a9c 100644 --- a/internal/metrics/sandbox_collector_test.go +++ b/internal/metrics/sandbox_collector_test.go @@ -170,7 +170,7 @@ func TestSandboxCollector(t *testing.T) { }, }, }, - expectedCount: 3, // We expect 3 distinct metric series for the 4 sandboxes + expectedCount: 3, expectedLabels: map[string]int{ "created_by:unknown expired:false launch_type:cold namespace:default owned_by:None ready_condition:true sandbox_template:unknown": 1, "created_by:unknown expired:true launch_type:warm namespace:test-ns owned_by:None ready_condition:false sandbox_template:my-template": 1, @@ -330,7 +330,7 @@ func TestSandboxCollector(t *testing.T) { for _, l := range m.GetLabel() { labelStr += l.GetName() + ":" + l.GetValue() + " " } - // Trim trailing space + if len(labelStr) > 0 { labelStr = labelStr[:len(labelStr)-1] } diff --git a/internal/metrics/tracing.go b/internal/metrics/tracing.go index c246d02..4e2542f 100644 --- a/internal/metrics/tracing.go +++ b/internal/metrics/tracing.go @@ -63,9 +63,7 @@ type otelInstrumenter struct { logger logr.Logger } -// StartSpan starts a span, potentially continuing one extracted from resource annotations. func (o *otelInstrumenter) StartSpan(ctx context.Context, obj metav1.Object, spanName string, attrs map[string]string) (context.Context, func()) { - // 1. Extract Parent Context from annotations if present. if obj != nil && obj.GetAnnotations() != nil { if tc, ok := obj.GetAnnotations()[TraceContextAnnotation]; ok && tc != "" { var carrier map[string]string @@ -77,7 +75,6 @@ func (o *otelInstrumenter) StartSpan(ctx context.Context, obj metav1.Object, spa } } - // 2. Prepare initial attributes (WithAttributes) opts := []trace.SpanStartOption{} if len(attrs) > 0 { otelAttrs := make([]attribute.KeyValue, 0, len(attrs)) @@ -87,12 +84,10 @@ func (o *otelInstrumenter) StartSpan(ctx context.Context, obj metav1.Object, spa opts = append(opts, trace.WithAttributes(otelAttrs...)) } - // 3. Start Span with options ctx, span := o.tracer.Start(ctx, spanName, opts...) return ctx, func() { span.End() } } -// GetTraceContext returns the current W3C context as a JSON string for persistence. func (o *otelInstrumenter) GetTraceContext(ctx context.Context) string { carrier := propagation.MapCarrier{} o.propagator.Inject(ctx, carrier) @@ -104,7 +99,6 @@ func (o *otelInstrumenter) GetTraceContext(ctx context.Context) string { return string(data) } -// AddEvent uses WithAttributes to provide info about state changes or progress. func (o *otelInstrumenter) AddEvent(ctx context.Context, name string, attrs map[string]string) { span := trace.SpanFromContext(ctx) otelAttrs := make([]attribute.KeyValue, 0, len(attrs)) @@ -115,7 +109,6 @@ func (o *otelInstrumenter) AddEvent(ctx context.Context, name string, attrs map[ span.AddEvent(name, trace.WithAttributes(otelAttrs...)) } -// Returns true if the span in the context is a real, sampled-in span. func (o *otelInstrumenter) IsRecording(ctx context.Context) bool { return trace.SpanFromContext(ctx).IsRecording() } @@ -136,7 +129,7 @@ func SetupOTel(ctx context.Context, serviceName string) (Instrumenter, func(), e )), ) otel.SetTracerProvider(tp) - // Use standard W3C Context propagator only (no Baggage). + // no Baggage: only the W3C trace context crosses the annotation. otel.SetTextMapPropagator(propagation.TraceContext{}) return &otelInstrumenter{ diff --git a/extensions/controllers/queue/simple_sandbox_queue.go b/internal/queue/simple_sandbox_queue.go similarity index 55% rename from extensions/controllers/queue/simple_sandbox_queue.go rename to internal/queue/simple_sandbox_queue.go index 9b5bfbd..8932e1f 100644 --- a/extensions/controllers/queue/simple_sandbox_queue.go +++ b/internal/queue/simple_sandbox_queue.go @@ -15,6 +15,7 @@ package queue import ( + "container/list" "sync" ) @@ -45,29 +46,28 @@ func (s *SimpleSandboxQueue) Add(namespacedWarmPoolName string, item SandboxKey) q.(*synchronizedQueue).Push(item) } -// Get pops an item from the specific warm pool's queue. -func (s *SimpleSandboxQueue) Get(namespacedWarmPoolName string) (SandboxKey, bool) { +// GetWithStrategy pops an item from the specific warm pool's queue using a custom strategy. +func (s *SimpleSandboxQueue) GetWithStrategy(namespacedWarmPoolName string, pick Strategy) (SandboxKey, bool) { q, ok := s.queues.Load(namespacedWarmPoolName) if !ok { return SandboxKey{}, false } - return q.(*synchronizedQueue).Pop() + return q.(*synchronizedQueue).PopWithStrategy(pick) } -// GetWithStrategy pops an item from the specific warm pool's queue using a custom strategy. -func (s *SimpleSandboxQueue) GetWithStrategy(namespacedWarmPoolName string, pick Strategy) (SandboxKey, bool) { +// Len reports how many sandboxes a warm pool's queue holds. +func (s *SimpleSandboxQueue) Len(namespacedWarmPoolName string) int { q, ok := s.queues.Load(namespacedWarmPoolName) if !ok { - return SandboxKey{}, false + return 0 } - return q.(*synchronizedQueue).PopWithStrategy(pick) + return q.(*synchronizedQueue).Len() } // RemoveItem deletes a specific sandbox from a warm pool's queue. func (s *SimpleSandboxQueue) RemoveItem(namespacedWarmPoolName string, item SandboxKey) { if q, ok := s.queues.Load(namespacedWarmPoolName); ok { - sq := q.(*synchronizedQueue) - sq.Remove(item) + q.(*synchronizedQueue).Remove(item) } } @@ -77,126 +77,83 @@ func (s *SimpleSandboxQueue) RemoveQueue(namespacedWarmPoolName string) { s.queues.Delete(namespacedWarmPoolName) } -// synchronizedQueue is one warm pool's FIFO of adoptable sandbox keys; +// synchronizedQueue is one warm pool's FIFO of adoptable sandbox keys, indexed +// by key so push and remove touch one element instead of scanning the pool; // RemoveQueue drops it when its SandboxWarmPool goes away. type synchronizedQueue struct { mu sync.Mutex - items []SandboxKey - set map[string]struct{} // Used for O(1) deduplication by namespace/name + order *list.List + items map[string]*list.Element } func newSynchronizedQueue() *synchronizedQueue { - return &synchronizedQueue{ - items: make([]SandboxKey, 0), - set: make(map[string]struct{}), - } + return &synchronizedQueue{order: list.New(), items: map[string]*list.Element{}} } -// Push adds an item to the queue if it isn't already present. +// Push appends an item, refreshing NodeName on a key already queued: placement +// may have settled since, and its arrival position must not change. func (q *synchronizedQueue) Push(key SandboxKey) { q.mu.Lock() defer q.mu.Unlock() - uniqueID := key.Namespace + "/" + key.Name - if _, exists := q.set[uniqueID]; !exists { - q.set[uniqueID] = struct{}{} - q.items = append(q.items, key) - } else { - // An existing key still refreshes NodeName: placement may have settled since. - for i := range q.items { - if q.items[i].Namespace == key.Namespace && q.items[i].Name == key.Name { - q.items[i].NodeName = key.NodeName - break - } - } + id := uniqueID(key) + if el, exists := q.items[id]; exists { + el.Value = key + return } + q.items[id] = q.order.PushBack(key) } -// Pop removes and returns the first item from the queue. -func (q *synchronizedQueue) Pop() (SandboxKey, bool) { +// PopWithStrategy removes and returns the key pick selects, in arrival order. +// pick is a pure in-memory function, so it runs under the lock: no snapshot to +// race against, and no re-verify retry. +func (q *synchronizedQueue) PopWithStrategy(pick Strategy) (SandboxKey, bool) { q.mu.Lock() defer q.mu.Unlock() - - if len(q.items) == 0 { + if q.order.Len() == 0 { return SandboxKey{}, false } - - item := q.items[0] - - // This removes the pointer references so the Garbage Collector - // can free the strings in memory! - q.items[0] = SandboxKey{} - - q.items = q.items[1:] - delete(q.set, item.Namespace+"/"+item.Name) - - return item, true -} - -// PopWithStrategy applies the strategy function to pick an item from the queue, -// removes it thread-safely, and returns it. -func (q *synchronizedQueue) PopWithStrategy(pick Strategy) (SandboxKey, bool) { - for { - q.mu.Lock() - if len(q.items) == 0 { - q.mu.Unlock() - return SandboxKey{}, false - } - - snapshot := make([]SandboxKey, len(q.items)) - copy(snapshot, q.items) - q.mu.Unlock() - - key, ok := pick(snapshot) - if !ok { - return SandboxKey{}, false - } - - q.mu.Lock() - uniqueID := key.Namespace + "/" + key.Name - // Verify the key is still present in the queue - if _, exists := q.set[uniqueID]; !exists { - // The picked key was concurrently popped by another goroutine. - // Unlock and retry snapshot and pick. - q.mu.Unlock() - continue - } - - q.removeLocked(key, uniqueID) - q.mu.Unlock() - - return key, true + keys := make([]SandboxKey, 0, q.order.Len()) + for el := q.order.Front(); el != nil; el = el.Next() { + keys = append(keys, el.Value.(SandboxKey)) + } + key, ok := pick(keys) + if !ok { + return SandboxKey{}, false } + q.removeLocked(uniqueID(key)) + return key, true } -// Remove scans the slice and deletes the item to prevent Ghost Pods. +// Remove deletes the item to prevent Ghost Pods. func (q *synchronizedQueue) Remove(key SandboxKey) { q.mu.Lock() defer q.mu.Unlock() + q.removeLocked(uniqueID(key)) +} - uniqueID := key.Namespace + "/" + key.Name - if _, exists := q.set[uniqueID]; !exists { - return - } +func (q *synchronizedQueue) Len() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.order.Len() +} - q.removeLocked(key, uniqueID) -} - -// removeLocked drops key's row and set entry, clearing the vacated tail slot -// so removed keys don't linger. Callers hold mu. -func (q *synchronizedQueue) removeLocked(key SandboxKey, uniqueID string) { - delete(q.set, uniqueID) - for i, k := range q.items { - if k.Namespace == key.Namespace && k.Name == key.Name { - last := len(q.items) - 1 - copy(q.items[i:], q.items[i+1:]) - q.items[last] = SandboxKey{} - q.items = q.items[:last] - break - } +// removeLocked drops one key's row and index entry. Callers hold mu. +func (q *synchronizedQueue) removeLocked(id string) { + el, exists := q.items[id] + if !exists { + return } + q.order.Remove(el) + delete(q.items, id) } // GetNamespacedWarmPoolName forms the namespace-aware index value to use as a key to a SimpleSandboxQueue type. func GetNamespacedWarmPoolName(namespace, warmPoolName string) string { return namespace + "/" + warmPoolName } + +// uniqueID identifies a queued sandbox independently of its placement, so a +// re-Add with a settled NodeName updates the row instead of duplicating it. +func uniqueID(key SandboxKey) string { + return key.Namespace + "/" + key.Name +} diff --git a/extensions/controllers/queue/simple_sandbox_queue_test.go b/internal/queue/simple_sandbox_queue_test.go similarity index 60% rename from extensions/controllers/queue/simple_sandbox_queue_test.go rename to internal/queue/simple_sandbox_queue_test.go index ef72c28..04229dd 100644 --- a/extensions/controllers/queue/simple_sandbox_queue_test.go +++ b/internal/queue/simple_sandbox_queue_test.go @@ -24,24 +24,20 @@ func TestSimpleSandboxQueue_BasicOperations(t *testing.T) { key1 := SandboxKey{Namespace: "default", Name: "sb-1"} key2 := SandboxKey{Namespace: "default", Name: "sb-2"} - // Test Add q.Add(hash, key1) q.Add(hash, key2) - // Test Get (Should be FIFO) - got1, ok1 := q.Get(hash) + got1, ok1 := pop(q, hash) if !ok1 || got1 != key1 { t.Errorf("Expected %v, got %v (ok: %v)", key1, got1, ok1) } - got2, ok2 := q.Get(hash) + got2, ok2 := pop(q, hash) if !ok2 || got2 != key2 { t.Errorf("Expected %v, got %v (ok: %v)", key2, got2, ok2) } - // Queue should now be empty - _, ok3 := q.Get(hash) - if ok3 { + if _, ok3 := pop(q, hash); ok3 { t.Errorf("Expected queue to be empty, but got an item") } } @@ -58,39 +54,23 @@ func TestSimpleSandboxQueue_RemoveItem_GhostPodFix(t *testing.T) { q.Add(hash, key2) q.Add(hash, key3) - // Simulate the Kubelet deleting the middle pod (Ghost Pod scenario) q.RemoveItem(hash, key2) - // Ensure RemoveItem does not retain stale references in backing array tail. - rawQueue, ok := q.queues.Load(hash) - if !ok { - t.Fatalf("Expected queue for %q to exist", hash) - } - sq := rawQueue.(*synchronizedQueue) - if cap(sq.items) > len(sq.items) { - backing := sq.items[:cap(sq.items)] - for i := len(sq.items); i < len(backing); i++ { - if backing[i] != (SandboxKey{}) { - t.Errorf("Expected backing array slot %d to be cleared, found %+v", i, backing[i]) - } - } + if got := q.Len(hash); got != 2 { + t.Errorf("Expected 2 items after removing the middle key, got %d", got) } - // First pop should still be key1 - got1, _ := q.Get(hash) + got1, _ := pop(q, hash) if got1 != key1 { t.Errorf("Expected %v, got %v", key1, got1) } - // Second pop should be key3! (key2 was successfully removed) - got3, _ := q.Get(hash) + got3, _ := pop(q, hash) if got3 != key3 { t.Errorf("Expected %v to skip deleted item and return %v, but got %v", hash, key3, got3) } - // Queue should now be empty - _, hasItem := q.Get(hash) - if hasItem { + if _, hasItem := pop(q, hash); hasItem { t.Errorf("Expected queue to be empty after Ghost Pod removal") } } @@ -99,19 +79,30 @@ func TestSynchronizedQueue_Deduplication(t *testing.T) { q := newSynchronizedQueue() key := SandboxKey{Namespace: "default", Name: "duplicate-sb"} - // Push the exact same pod 3 times q.Push(key) q.Push(key) q.Push(key) - // Verify it only stored it once - if len(q.items) != 1 { - t.Errorf("Expected length 1 due to O(1) deduplication, got %d", len(q.items)) + if got := q.Len(); got != 1 { + t.Errorf("Expected length 1 due to O(1) deduplication, got %d", got) } + if got := len(q.items); got != 1 { + t.Errorf("Expected index length 1, got %d", got) + } +} - // Verify the set also only has 1 item - if len(q.set) != 1 { - t.Errorf("Expected set length 1, got %d", len(q.set)) +func TestSynchronizedQueue_PushKeepsArrivalPositionAndRefreshesNode(t *testing.T) { + q := newSynchronizedQueue() + first := SandboxKey{Namespace: "default", Name: "sb-1"} + second := SandboxKey{Namespace: "default", Name: "sb-2"} + + q.Push(first) + q.Push(second) + q.Push(SandboxKey{Namespace: "default", Name: "sb-1", NodeName: "node-a"}) + + got, ok := q.PopWithStrategy(func(keys []SandboxKey) (SandboxKey, bool) { return keys[0], true }) + if !ok || got.Name != "sb-1" || got.NodeName != "node-a" { + t.Errorf("Expected sb-1 first with a refreshed node, got %+v (ok: %v)", got, ok) } } @@ -121,13 +112,9 @@ func TestSimpleSandboxQueue_RemoveQueue_MemoryLeakFix(t *testing.T) { key1 := SandboxKey{Namespace: "default", Name: "sb-1"} q.Add(hash, key1) - - // Simulate SandboxTemplate deletion q.RemoveQueue(hash) - // Verify the entire queue was wiped from the sync.Map - _, ok := q.Get(hash) - if ok { + if _, ok := pop(q, hash); ok { t.Errorf("Expected queue to be completely removed, but it still existed") } } @@ -144,7 +131,6 @@ func TestSimpleSandboxQueue_GetWithStrategy(t *testing.T) { q.Add(hash, key2) q.Add(hash, key3) - // Custom strategy to pick key2 specifically pickKey2 := func(items []SandboxKey) (SandboxKey, bool) { for _, item := range items { if item.Name == "sb-2" { @@ -154,27 +140,22 @@ func TestSimpleSandboxQueue_GetWithStrategy(t *testing.T) { return SandboxKey{}, false } - // Pop with strategy got, ok := q.GetWithStrategy(hash, pickKey2) if !ok || got != key2 { t.Errorf("Expected to pick %v, got %v (ok: %v)", key2, got, ok) } - // First standard pop should be key1 (since key2 was removed) - got1, _ := q.Get(hash) + got1, _ := pop(q, hash) if got1 != key1 { t.Errorf("Expected first remaining item to be %v, got %v", key1, got1) } - // Second standard pop should be key3 - got3, _ := q.Get(hash) + got3, _ := pop(q, hash) if got3 != key3 { t.Errorf("Expected second remaining item to be %v, got %v", key3, got3) } - // Queue should now be empty - _, ok3 := q.Get(hash) - if ok3 { + if _, ok3 := pop(q, hash); ok3 { t.Errorf("Expected queue to be empty, but got an item") } } @@ -196,40 +177,29 @@ func TestSimpleSandboxQueue_NoLegacyFallback(t *testing.T) { namespacedName := GetNamespacedWarmPoolName(namespace, wpName) key1 := SandboxKey{Namespace: namespace, Name: "sb-1"} - - // Store queue with namespace-aware warm pool name q.Add(namespacedName, key1) - // Verify that namespace-agnostic warm pool name does NOT work to Get - _, ok := q.Get(wpName) - if ok { - t.Errorf("Expected Get with namespace-agnostic name to fail") + if _, ok := pop(q, wpName); ok { + t.Errorf("Expected pop with namespace-agnostic name to fail") } - // Verify that namespace-agnostic warm pool name does NOT work to GetWithStrategy - _, ok = q.GetWithStrategy(wpName, func(items []SandboxKey) (SandboxKey, bool) { - return items[0], true - }) - if ok { + if _, ok := q.GetWithStrategy(wpName, func(items []SandboxKey) (SandboxKey, bool) { return items[0], true }); ok { t.Errorf("Expected GetWithStrategy with namespace-agnostic name to fail") } - // Verify that namespace-agnostic warm pool name does NOT work to RemoveItem q.RemoveItem(wpName, key1) - // We use GetWithStrategy without popping, or check queue length by standard Get. - // Since Get pops, let's check that Get(namespacedName) still succeeds and returns key1. - got, ok := q.Get(namespacedName) - if !ok || got != key1 { - t.Errorf("Expected item to still be in queue after RemoveItem with namespace-agnostic name") + if got := q.Len(namespacedName); got != 1 { + t.Errorf("Expected item to still be queued after RemoveItem with namespace-agnostic name, got %d", got) } - // Re-add item since Get popped it - q.Add(namespacedName, key1) - - // Verify that namespace-agnostic warm pool name does NOT work to RemoveQueue q.RemoveQueue(wpName) - _, ok = q.Get(namespacedName) - if !ok { - t.Errorf("Expected queue to still exist after RemoveQueue with namespace-agnostic name") + if got := q.Len(namespacedName); got != 1 { + t.Errorf("Expected queue to still exist after RemoveQueue with namespace-agnostic name, got %d", got) } } + +func pop(q *SimpleSandboxQueue, namespacedWarmPoolName string) (SandboxKey, bool) { + return q.GetWithStrategy(namespacedWarmPoolName, func(keys []SandboxKey) (SandboxKey, bool) { + return keys[0], true + }) +} diff --git a/internal/version/version.go b/internal/version/version.go index ed80035..26037f1 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -16,7 +16,6 @@ package version import ( - "bytes" "fmt" "runtime" "strings" @@ -52,6 +51,11 @@ type Info struct { Platform string `json:"platform"` } +// String returns a Go-syntax representation of the Info. +func (info Info) String() string { + return fmt.Sprintf("%#v", info) +} + // Get returns version information populated with build-time values. func Get() Info { return Info{ @@ -64,18 +68,13 @@ func Get() Info { } } -// String returns a Go-syntax representation of the Info. -func (info Info) String() string { - return fmt.Sprintf("%#v", info) -} - // Print returns version information. func Print(program string) string { m := Get() m.Program = program t := template.Must(template.New("version").Parse(versionInfoTmpl)) - var buf bytes.Buffer + var buf strings.Builder if err := t.ExecuteTemplate(&buf, "version", m); err != nil { panic(err) } diff --git a/k8s/crds/extensions.agents.x-k8s.io_nodeinventories.yaml b/k8s/crds/extensions.agents.x-k8s.io_nodeinventories.yaml index 772f6c5..f2ae849 100644 --- a/k8s/crds/extensions.agents.x-k8s.io_nodeinventories.yaml +++ b/k8s/crds/extensions.agents.x-k8s.io_nodeinventories.yaml @@ -45,6 +45,8 @@ spec: type: string phase: type: string + template: + type: string required: - name - phase diff --git a/pkg/e2bcompat/http.go b/pkg/e2bcompat/http.go index af0b5bc..ce9de5e 100644 --- a/pkg/e2bcompat/http.go +++ b/pkg/e2bcompat/http.go @@ -11,10 +11,6 @@ const ( // maxBodyBytes caps a request body. The e2b bodies are small objects; the // cap keeps an unbounded upload off the claim path. maxBodyBytes = 1 << 20 - // tokenAnnotation carries the per-sandbox ownership credential the claim - // returned. It mirrors the aggregated apiserver's annotation of the same - // name, and is surfaced to the SDK as envdAccessToken. - tokenAnnotation = "sandbox.cocoonstack.io/token" // namePrefix prefixes the Kubernetes object name of a compat claim, so a // sandbox created through this surface is recognizable in `kubectl get // sandboxes` and in node inventory. diff --git a/pkg/e2bcompat/lifecycle.go b/pkg/e2bcompat/lifecycle.go index 7c25a4a..debe150 100644 --- a/pkg/e2bcompat/lifecycle.go +++ b/pkg/e2bcompat/lifecycle.go @@ -7,12 +7,20 @@ import ( "fmt" "io" "net/http" + "slices" + "sync/atomic" "time" + "golang.org/x/sync/errgroup" + sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" "github.com/cocoonstack/sandbox-operator/pkg/scale" ) +// maxNodeConcurrency bounds a fleet-wide fan-out so a handful of wedged nodes +// cannot serialize a handler into the minutes. +const maxNodeConcurrency = 16 + // pauseSandbox hibernates the sandbox: its memory is written out and the VM // stops, so the cost is proportional to guest RAM. e2b's contract is specific // about the already-paused case — the SDK reads 409 as "already paused" and @@ -76,13 +84,14 @@ func (s *Server) connectSandbox(w http.ResponseWriter, r *http.Request) { } status = http.StatusCreated } + // envdAccessToken is left empty: the token is handed out once at claim time + // and node inventory carries no per-sandbox secret to re-derive it from. writeJSON(w, status, Sandbox{ - TemplateID: templateOf(sb), - SandboxID: publicID(claimIDOf(sb)), - ClientID: sb.Status.NodeName, - EnvdVersion: s.opts.EnvdVersion, - EnvdAccessToken: sb.Annotations[tokenAnnotation], - Domain: s.opts.Domain, + TemplateID: templateOf(sb), + SandboxID: publicID(claimIDOf(sb)), + ClientID: sb.Status.NodeName, + EnvdVersion: s.opts.EnvdVersion, + Domain: s.opts.Domain, }) } @@ -129,8 +138,7 @@ func (s *Server) forkSandbox(w http.ResponseWriter, r *http.Request) { } template := templateOf(sb) out := make([]SandboxForkResult, 0, len(children)) - for i := range children { - child := children[i] + for _, child := range children { out = append(out, SandboxForkResult{Sandbox: &Sandbox{ TemplateID: template, SandboxID: publicID(child.SandboxName), @@ -174,17 +182,27 @@ func (s *Server) listSnapshots(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "failed to list snapshots") return } - out := []SnapshotInfo{} - for _, node := range nodes { - snaps, err := s.store.Snapshots(r.Context(), node) - if err != nil { - // One unreachable node must not blank the whole listing. - s.opts.Log.Error(err, "e2b list snapshots: node failed", "node", node) - continue - } - for _, snap := range snaps { - out = append(out, snapshotInfo(snap)) - } + perNode := make([][]SnapshotInfo, len(nodes)) + var g errgroup.Group + g.SetLimit(maxNodeConcurrency) + for i, node := range nodes { + g.Go(func() error { + snaps, err := s.store.Snapshots(r.Context(), node) + if err != nil { + // One unreachable node must not blank the whole listing. + s.opts.Log.Error(err, "e2b list snapshots: node failed", "node", node) + return nil + } + for _, snap := range snaps { + perNode[i] = append(perNode[i], snapshotInfo(snap)) + } + return nil + }) + } + _ = g.Wait() + out := slices.Concat(perNode...) + if out == nil { + out = []SnapshotInfo{} } writeJSON(w, http.StatusOK, out) } @@ -202,10 +220,27 @@ func (s *Server) deleteSnapshot(w http.ResponseWriter, r *http.Request) { // Checkpoints are node-local and the id does not name its node, so the // delete is offered to each node; a node that does not hold it reports // success (delete is idempotent), which keeps this safe to fan out. + var ( + g errgroup.Group + ok atomic.Bool + ) + g.SetLimit(maxNodeConcurrency) for _, node := range nodes { - if err := s.store.DeleteSnapshot(r.Context(), node, snapshotID); err != nil { - s.opts.Log.Error(err, "e2b delete snapshot: node failed", "node", node, "snapshotID", snapshotID) - } + g.Go(func() error { + if err := s.store.DeleteSnapshot(r.Context(), node, snapshotID); err != nil { + s.opts.Log.Error(err, "e2b delete snapshot: node failed", "node", node, "snapshotID", snapshotID) + return nil + } + ok.Store(true) + return nil + }) + } + _ = g.Wait() + // Every node failing is an outage, not an idempotent delete of something + // already gone, so the caller must not read it as success. + if len(nodes) > 0 && !ok.Load() { + writeError(w, http.StatusInternalServerError, "failed to delete the snapshot on any node") + return } w.WriteHeader(http.StatusNoContent) } diff --git a/pkg/e2bcompat/lifecycle_test.go b/pkg/e2bcompat/lifecycle_test.go index 1989e6a..79d3c36 100644 --- a/pkg/e2bcompat/lifecycle_test.go +++ b/pkg/e2bcompat/lifecycle_test.go @@ -10,14 +10,10 @@ import ( "github.com/cocoonstack/sandbox-operator/pkg/scale" ) -// TestPauseAlreadyPausedIs409 is load-bearing for the SDK, not cosmetic: -// e2b's pause() returns a boolean, and it derives false ("was already paused") -// from a 409. Reporting anything else turns an idempotent no-op into an error -// the caller has to interpret. func TestPauseAlreadyPausedIs409(t *testing.T) { store := &lifecycleStore{} nodeReportsPaused(store) - store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", ``, testKey) @@ -29,11 +25,9 @@ func TestPauseAlreadyPausedIs409(t *testing.T) { } } -// TestPauseRoutesToOwningNode: the verb must address the node-local claim id on -// the owning node, never the public id the client spelled. func TestPauseRoutesToOwningNode(t *testing.T) { store := &lifecycleStore{} - store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", ``, testKey) @@ -46,13 +40,9 @@ func TestPauseRoutesToOwningNode(t *testing.T) { } } -// TestPauseFilesystemOnlyIsRejected: e2b's memory=false asks for a -// filesystem-only snapshot whose resume cold-boots. The node always captures -// memory, so honoring the flag would hand back a different sandbox than the -// caller asked for — say so instead of silently doing the other thing. func TestPauseFilesystemOnlyIsRejected(t *testing.T) { store := &lifecycleStore{} - store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/pause", `{"memory":false}`, testKey) @@ -64,13 +54,9 @@ func TestPauseFilesystemOnlyIsRejected(t *testing.T) { } } -// TestConnectRunningIs200 / TestConnectPausedIs201: e2b's connect IS the SDK's -// resume. 200 means it was already running, 201 that a restore actually -// happened — the SDK accepts either, and the distinction is what tells an -// operator whether the mmap restore path ran. func TestConnectRunningIs200(t *testing.T) { store := &lifecycleStore{} - store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/connect", `{"timeout":30}`, testKey) @@ -92,7 +78,7 @@ func TestConnectRunningIs200(t *testing.T) { func TestConnectPausedIs201AndResumes(t *testing.T) { store := &lifecycleStore{} nodeReportsPaused(store) - store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/connect", `{"timeout":30}`, testKey) @@ -104,12 +90,10 @@ func TestConnectPausedIs201AndResumes(t *testing.T) { } } -// TestForkPausedIs409 mirrors e2b: a paused source cannot be forked, and the -// message must say to resume it first rather than failing opaquely. func TestForkPausedIs409(t *testing.T) { store := &lifecycleStore{} nodeReportsPaused(store) - store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{pausedSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/fork", `{"count":2}`, testKey) @@ -121,9 +105,6 @@ func TestForkPausedIs409(t *testing.T) { } } -// TestForkReturnsPerChildResults: e2b's fork reply is one entry per child, each -// carrying its own new sandbox id — children are fresh sandboxes, not replicas -// of the parent's identity. func TestForkReturnsPerChildResults(t *testing.T) { store := &lifecycleStore{ forkChildren: []scale.Assignment{ @@ -131,7 +112,7 @@ func TestForkReturnsPerChildResults(t *testing.T) { {SandboxName: "sb_c2", Node: "node-a", Token: "t2"}, }, } - store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/fork", `{"count":2}`, testKey) @@ -155,10 +136,9 @@ func TestForkReturnsPerChildResults(t *testing.T) { } } -// TestForkDefaultsToOneChild: count is optional in e2b's schema. func TestForkDefaultsToOneChild(t *testing.T) { store := &lifecycleStore{forkChildren: []scale.Assignment{{SandboxName: "sb_c1", Node: "node-a"}}} - store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) if w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/fork", ``, testKey); w.Code != http.StatusCreated { @@ -169,11 +149,9 @@ func TestForkDefaultsToOneChild(t *testing.T) { } } -// TestSnapshotReturns201WithID: the source keeps running; the reply carries the -// checkpoint id a later branch is taken from. func TestSnapshotReturns201WithID(t *testing.T) { store := &lifecycleStore{snapshot: scale.Snapshot{ID: "ck_1234", Name: "before-migration", Node: "node-a"}} - store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img", "tok")} + store.items = []sandboxv1beta1.Sandbox{liveSandbox("s1", "sb_abc", "node-a", "img")} h := newTestServer(t, store) w := do(t, h, http.MethodPost, "/sandboxes/sb-abc/snapshots", `{"name":"before-migration"}`, testKey) @@ -195,8 +173,6 @@ func TestSnapshotReturns201WithID(t *testing.T) { } } -// TestLifecycleVerbsOnUnknownSandboxAre404 keeps the SDK's not-found branches -// working: it raises SandboxNotFound on 404 rather than a generic error. func TestLifecycleVerbsOnUnknownSandboxAre404(t *testing.T) { for _, path := range []string{ "/sandboxes/sb-missing/pause", @@ -213,18 +189,11 @@ func TestLifecycleVerbsOnUnknownSandboxAre404(t *testing.T) { } } -// TestPauseTrustsTheNodeNotTheStaleView reproduces a bug found in the cluster: -// the read view is synthesized from NodeInventory, which the node republishes -// on a ~30 s cadence, so for up to half a minute after a pause the listed -// sandbox still carries phase=Running. Deciding "already paused" from that -// label let a second pause through with 204, telling the caller it had paused a -// sandbox that was already down — and the e2b SDK turns that 204 into "yes, I -// paused it". The owning node is authoritative and must win. func TestPauseTrustsTheNodeNotTheStaleView(t *testing.T) { store := &lifecycleStore{} - store.nodePaused = true // the node has it hibernated ... - sb := liveSandbox("s1", "sb_abc", "node-a", "img", "tok") - sb.Labels = map[string]string{scale.PhaseLabel: "Running"} // ... but the view still says Running + store.nodePaused = true + sb := liveSandbox("s1", "sb_abc", "node-a", "img") + sb.Labels = map[string]string{scale.PhaseLabel: "Running"} store.items = []sandboxv1beta1.Sandbox{sb} h := newTestServer(t, store) @@ -237,13 +206,10 @@ func TestPauseTrustsTheNodeNotTheStaleView(t *testing.T) { } } -// TestConnectTrustsTheNodeNotTheStaleView is the same hazard on resume: a stale -// Running label would return 200 without restoring, handing the caller -// connection details for a VM that is not running. func TestConnectTrustsTheNodeNotTheStaleView(t *testing.T) { store := &lifecycleStore{} store.nodePaused = true - sb := liveSandbox("s1", "sb_abc", "node-a", "img", "tok") + sb := liveSandbox("s1", "sb_abc", "node-a", "img") sb.Labels = map[string]string{scale.PhaseLabel: "Running"} store.items = []sandboxv1beta1.Sandbox{sb} h := newTestServer(t, store) @@ -257,8 +223,6 @@ func TestConnectTrustsTheNodeNotTheStaleView(t *testing.T) { } } -// lifecycleStore records which verb the compat layer routed where, so the tests -// assert the translation rather than any node behavior. type lifecycleStore struct { fakeStore @@ -271,8 +235,6 @@ type lifecycleStore struct { snapshot scale.Snapshot err error - // nodePaused is what the owning node reports for Stats().Paused — the - // authoritative answer isPaused now consults instead of the cached label. nodePaused bool } @@ -300,13 +262,10 @@ func (f *lifecycleStore) Snapshot(_ context.Context, _, _, name string) (scale.S return f.snapshot, f.err } -// pausedSandbox is a sandbox as the node publishes it while hibernated. -func pausedSandbox(name, claimID, node, image, token string) sandboxv1beta1.Sandbox { - sb := liveSandbox(name, claimID, node, image, token) - sb.Labels = map[string]string{scale.PhaseLabel: phaseHibernated} +func pausedSandbox(name, claimID, node, template string) sandboxv1beta1.Sandbox { + sb := liveSandbox(name, claimID, node, template) + sb.Labels[scale.PhaseLabel] = phaseHibernated return sb } -// nodeReportsPaused makes the fake's owning node report the sandbox hibernated, -// which is what isPaused actually consults. func nodeReportsPaused(s *lifecycleStore) { s.nodePaused = true } diff --git a/pkg/e2bcompat/lookup_bench_test.go b/pkg/e2bcompat/lookup_bench_test.go index 21df01d..7175de1 100644 --- a/pkg/e2bcompat/lookup_bench_test.go +++ b/pkg/e2bcompat/lookup_bench_test.go @@ -8,10 +8,6 @@ import ( "github.com/cocoonstack/sandbox-operator/pkg/scale" ) -// BenchmarkLookupByID resolves one sandbox id living on the last node at the -// 200x2000 fleet projection — the per-request cost of every single-sandbox e2b -// verb (get/delete/pause/connect/fork/snapshot/metrics). The requested id uses -// the published DNS-safe form, the slower of the two accepted spellings. func BenchmarkLookupByID(b *testing.B) { const nodes, perNode = 200, 2000 src := scale.NewStaticInventorySource() diff --git a/pkg/e2bcompat/sandboxid.go b/pkg/e2bcompat/sandboxid.go index dcf028c..831cb79 100644 --- a/pkg/e2bcompat/sandboxid.go +++ b/pkg/e2bcompat/sandboxid.go @@ -5,19 +5,10 @@ import ( "unicode/utf8" ) -// A sandboxd claim id is "sb_" + hex (sandboxd pool/claim.go), whose underscore -// is not legal in a DNS label. The e2b SDK derives the in-sandbox envd host as -// "{port}-{sandboxID}.{domain}", so an id carrying an underscore produces a host -// that cannot resolve — the sandbox would be created but unreachable. -// -// The compat surface therefore publishes a DNS-safe rendering of the claim id -// and accepts either form on the way back in. The mapping only rewrites -// characters that are illegal in a DNS label, so it is stable, and it round -// trips for every id sandboxd actually mints (whose only illegal character is -// that one underscore). - // publicID renders a node-local claim id as a DNS-label-safe sandbox id, the -// form handed to e2b clients. +// form handed to e2b clients. A claim id is "sb_" + hex, and the SDK derives +// the envd host as "{port}-{sandboxID}.{domain}", so the underscore would make +// a created sandbox unreachable. func publicID(claimID string) string { if !needsRewrite(claimID) { return claimID diff --git a/pkg/e2bcompat/sandboxid_test.go b/pkg/e2bcompat/sandboxid_test.go index dd798a9..3521b1e 100644 --- a/pkg/e2bcompat/sandboxid_test.go +++ b/pkg/e2bcompat/sandboxid_test.go @@ -2,11 +2,6 @@ package e2bcompat import "testing" -// TestPublicIDIsDNSLabelSafe is the reason this mapping exists: the e2b SDK -// builds the in-sandbox envd host as "{port}-{sandboxID}.{domain}". A sandboxd -// claim id is "sb_"+hex, and that underscore is illegal in a DNS label — an id -// published raw yields a host that cannot resolve, so the sandbox is created -// but unreachable. func TestPublicIDIsDNSLabelSafe(t *testing.T) { for _, tt := range []struct { name string @@ -32,9 +27,6 @@ func TestPublicIDIsDNSLabelSafe(t *testing.T) { } } -// TestMatchesIDAcceptsBothForms: an id observed through either surface has to -// keep working. A client that saw the published (DNS-safe) id sends that back; -// something reading the raw claim id from the node sends the original. func TestMatchesIDAcceptsBothForms(t *testing.T) { const claim = "sb_0123456789abcdef" @@ -49,9 +41,6 @@ func TestMatchesIDAcceptsBothForms(t *testing.T) { } } -// TestMatchesIDRejectsEmpty guards the lookup loop: a sandbox whose node has -// not published a claim id has an empty annotation, and an empty request path -// is equally meaningless. Matching those would return an arbitrary sandbox. func TestMatchesIDRejectsEmpty(t *testing.T) { if matchesID("", "") { t.Error("empty must not match empty") diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index 0ede211..d47a2c9 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -20,7 +20,6 @@ package e2bcompat import ( - "context" "crypto/subtle" "encoding/json" "errors" @@ -50,7 +49,10 @@ const ( phaseHibernated = "Hibernated" ) -var errSandboxNotFound = errors.New("sandbox not found") +var ( + errSandboxNotFound = errors.New("sandbox not found") + errNoOwningNode = errors.New("sandbox inventory entry names no owning node") +) // Options configures the compat server. type Options struct { @@ -85,26 +87,26 @@ type Options struct { Log logr.Logger } -// claimIDResolver is the store fast path resolving one sandbox by node-local -// claim id without materializing the fleet; the scatter-gather store -// implements it, and lookup falls back to a List scan for stores that don't. -type claimIDResolver interface { - GetByClaimID(ctx context.Context, namespace string, match func(claimID string) bool) (*sandboxv1beta1.Sandbox, error) -} - // Server translates e2b REST calls onto a scale.SandboxStore. type Server struct { - store scale.SandboxStore - opts Options - keys map[string]struct{} + store scale.SandboxStore + resolver scale.ClaimIDResolver + opts Options + keys map[string]struct{} } // NewServer builds a compat server. It fails when no API key is configured and -// anonymous access was not explicitly allowed. +// anonymous access was not explicitly allowed, or when store cannot resolve a +// sandbox by claim id — every by-id verb would otherwise degrade to a +// cluster-wide List scan. func NewServer(store scale.SandboxStore, opts Options) (*Server, error) { if store == nil { return nil, errors.New("e2bcompat: store is required") } + resolver, ok := store.(scale.ClaimIDResolver) + if !ok { + return nil, errors.New("e2bcompat: store does not implement scale.ClaimIDResolver") + } if opts.Namespace == "" { opts.Namespace = "default" } @@ -123,7 +125,7 @@ func NewServer(store scale.SandboxStore, opts Options) (*Server, error) { if len(keys) == 0 && !opts.AllowAnonymous { return nil, errors.New("e2bcompat: no API key configured; set one or enable anonymous access explicitly") } - return &Server{store: store, opts: opts, keys: keys}, nil + return &Server{store: store, resolver: resolver, opts: opts, keys: keys}, nil } // Handler returns the routed, authenticated HTTP handler. @@ -266,8 +268,8 @@ func (s *Server) deleteSandbox(w http.ResponseWriter, r *http.Request) { } node := sb.Status.NodeName if node == "" { - // No owning node resolved: nothing to release against. - w.WriteHeader(http.StatusNoContent) + s.opts.Log.Error(errNoOwningNode, "e2b delete: release failed", "sandboxID", id) + writeError(w, http.StatusInternalServerError, "failed to release the sandbox") return } // Release against the raw node-local claim id, never the id as the client @@ -320,28 +322,16 @@ func (s *Server) lookup(r *http.Request, id string) (*sandboxv1beta1.Sandbox, er if strings.TrimSpace(id) == "" { return nil, errSandboxNotFound } - if resolver, ok := s.store.(claimIDResolver); ok { - sb, err := resolver.GetByClaimID(r.Context(), s.opts.Namespace, func(claimID string) bool { - return matchesID(claimID, id) - }) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil, errSandboxNotFound - } - return nil, err - } - return sb, nil - } - list, err := s.store.List(r.Context(), scale.ListOptions{Namespace: s.opts.Namespace}) + sb, err := s.resolver.GetByClaimID(r.Context(), s.opts.Namespace, id, func(claimID string) bool { + return matchesID(claimID, id) + }) if err != nil { - return nil, err - } - for i := range list.Items { - if matchesID(list.Items[i].Annotations[scale.ClaimIDAnnotation], id) { - return &list.Items[i], nil + if k8serrors.IsNotFound(err) { + return nil, errSandboxNotFound } + return nil, err } - return nil, errSandboxNotFound + return sb, nil } func (s *Server) writeLookupError(w http.ResponseWriter, err error, id, op string) { @@ -355,7 +345,9 @@ func (s *Server) writeLookupError(w http.ResponseWriter, err error, id, op strin // detailFor renders a live Sandbox as the e2b detail shape. Fields e2b requires // but cocoon does not track per sandbox (disk size) are reported as zero values -// rather than omitted, so the SDK's decoder stays happy. +// rather than omitted, so the SDK's decoder stays happy. envdAccessToken is one +// of them on this path: the token is handed out once at claim time and node +// inventory deliberately carries no per-sandbox secret. func (s *Server) detailFor(sb *sandboxv1beta1.Sandbox) SandboxDetail { started := sb.CreationTimestamp.Time if started.IsZero() { @@ -370,21 +362,24 @@ func (s *Server) detailFor(sb *sandboxv1beta1.Sandbox) SandboxDetail { endAt = deadline } return SandboxDetail{ - TemplateID: templateOf(sb), - SandboxID: publicID(sb.Annotations[scale.ClaimIDAnnotation]), - ClientID: sb.Status.NodeName, - StartedAt: started.UTC().Format(time.RFC3339), - EndAt: endAt.UTC().Format(time.RFC3339), - State: state, - EnvdVersion: s.opts.EnvdVersion, - EnvdAccessToken: sb.Annotations[tokenAnnotation], - Domain: s.opts.Domain, + TemplateID: templateOf(sb), + SandboxID: publicID(sb.Annotations[scale.ClaimIDAnnotation]), + ClientID: sb.Status.NodeName, + StartedAt: started.UTC().Format(time.RFC3339), + EndAt: endAt.UTC().Format(time.RFC3339), + State: state, + EnvdVersion: s.opts.EnvdVersion, + Domain: s.opts.Domain, } } -// templateOf reports the pool template a sandbox was claimed from: the first -// container image, the same axis scale.PoolKeyFor keys on. +// templateOf reports the pool template a sandbox was claimed from. A sandbox +// synthesized from node inventory carries it as a label; only an object that +// still holds its own pod spec can be read for the container image. func templateOf(sb *sandboxv1beta1.Sandbox) string { + if t := sb.Labels[scale.TemplateLabel]; t != "" { + return t + } if c := sb.Spec.PodTemplate.Spec.Containers; len(c) > 0 { return c[0].Image } diff --git a/pkg/e2bcompat/server_test.go b/pkg/e2bcompat/server_test.go index 0078bb0..1b9ae43 100644 --- a/pkg/e2bcompat/server_test.go +++ b/pkg/e2bcompat/server_test.go @@ -10,7 +10,7 @@ import ( "time" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -20,9 +20,6 @@ import ( const testKey = "e2b_testkey" -// TestCreateClaimsFromTemplatePool is the core contract: an e2b create is the -// same node-local claim, keyed on the template the caller asked for, and the -// response carries the fields the SDK requires. func TestCreateClaimsFromTemplatePool(t *testing.T) { store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_abc123", Node: "node-a", Token: "tok-1"}} h := newTestServer(t, store, func(o *Options) { o.Domain = "sandbox.example.com" }) @@ -35,9 +32,7 @@ func TestCreateClaimsFromTemplatePool(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { t.Fatalf("decode response: %v", err) } - // The published id is the DNS-label-safe rendering of the node's claim id, - // not the raw one: the SDK builds "{port}-{sandboxID}.{domain}", and the - // claim id's underscore would make that host unresolvable. + if got.SandboxID != "sb-abc123" { t.Errorf("sandboxID = %q, want the DNS-safe rendering of the claim id", got.SandboxID) } @@ -50,8 +45,7 @@ func TestCreateClaimsFromTemplatePool(t *testing.T) { if got.Domain != "sandbox.example.com" { t.Errorf("domain = %q, want the configured domain", got.Domain) } - // The SDK version-compares envdVersion and kills the sandbox if it cannot - // parse it or it is below 0.1.0, so it must always be a real version. + if got.EnvdVersion == "" { t.Error("envdVersion is empty; the e2b SDK kills the sandbox and throws when it cannot parse this") } @@ -69,7 +63,6 @@ func TestCreateClaimsFromTemplatePool(t *testing.T) { } } -// TestCreateNetworkLane checks e2b's internet flag selects the pool's net axis. func TestCreateNetworkLane(t *testing.T) { for _, tc := range []struct { name string @@ -93,8 +86,6 @@ func TestCreateNetworkLane(t *testing.T) { } } -// TestCreateNoWarmCapacityIsRetryable keeps the retryable signal retryable: a -// drained pool refills asynchronously, so it must not surface as a 500. func TestCreateNoWarmCapacityIsRetryable(t *testing.T) { store := &fakeStore{claimErr: scale.ErrNoWarmCapacity} h := newTestServer(t, store) @@ -120,8 +111,6 @@ func TestCreateRejectsNegativeTimeout(t *testing.T) { } } -// TestAuthRequiresAPIKey proves the claim endpoint is closed without the key -// the SDK presents. func TestAuthRequiresAPIKey(t *testing.T) { store := &fakeStore{assign: scale.Assignment{SandboxName: "sb_1", Node: "n"}} h := newTestServer(t, store) @@ -142,8 +131,6 @@ func TestAuthRequiresAPIKey(t *testing.T) { } } -// TestNewServerRefusesOpenByDefault: forgetting to configure a key must fail -// loud at startup, not silently serve an open claim endpoint. func TestNewServerRefusesOpenByDefault(t *testing.T) { if _, err := NewServer(&fakeStore{}, Options{Namespace: "x"}); err == nil { t.Fatal("NewServer accepted no API key without explicit anonymous access") @@ -160,12 +147,10 @@ func TestHealthNeedsNoKey(t *testing.T) { } } -// TestDeleteReleasesTheClaimedID checks release targets the node-local claim id -// rather than the Kubernetes name — releasing by name would free the wrong VM. func TestDeleteReleasesTheClaimedID(t *testing.T) { store := &fakeStore{items: []sandboxv1beta1.Sandbox{ - liveSandbox("e2b-aaa", "sb_one", "node-a", "img:1", "tok"), - liveSandbox("e2b-bbb", "sb_two", "node-b", "img:1", "tok"), + liveSandbox("e2b-aaa", "sb_one", "node-a", "img:1"), + liveSandbox("e2b-bbb", "sb_two", "node-b", "img:1"), }} h := newTestServer(t, store) @@ -178,7 +163,7 @@ func TestDeleteReleasesTheClaimedID(t *testing.T) { } func TestDeleteUnknownSandboxIs404(t *testing.T) { - store := &fakeStore{items: []sandboxv1beta1.Sandbox{liveSandbox("a", "sb_one", "node-a", "i", "t")}} + store := &fakeStore{items: []sandboxv1beta1.Sandbox{liveSandbox("a", "sb_one", "node-a", "i")}} h := newTestServer(t, store) if w := do(t, h, http.MethodDelete, "/sandboxes/sb_missing", "", testKey); w.Code != http.StatusNotFound { @@ -189,23 +174,22 @@ func TestDeleteUnknownSandboxIs404(t *testing.T) { } } -// TestGetReportsDetail checks the detail shape the SDK decodes on getInfo. func TestGetReportsDetail(t *testing.T) { - got := getDetail(t, liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04", "tok-9")) + got := getDetail(t, liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04")) if got.SandboxID != "sb-one" || got.TemplateID != "registry/rt:24.04" || got.ClientID != "node-a" { t.Errorf("detail = %+v, want the live sandbox's id/template/node", got) } if got.State != StateRunning { t.Errorf("state = %q, want %q", got.State, StateRunning) } - // The SDK parses these as dates; empty strings make it produce Invalid Date. + if got.StartedAt == "" || got.EndAt == "" { t.Errorf("startedAt/endAt = %q/%q, want RFC3339 timestamps", got.StartedAt, got.EndAt) } } func TestGetReportsTheGrantedDeadlineAsEndAt(t *testing.T) { - sb := liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04", "tok-9") + sb := liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04") sb.Annotations[scale.DeadlineAnnotation] = "2030-01-02T03:04:05Z" if got := getDetail(t, sb); got.EndAt != "2030-01-02T03:04:05Z" { t.Errorf("endAt = %q, want the granted deadline", got.EndAt) @@ -213,18 +197,17 @@ func TestGetReportsTheGrantedDeadlineAsEndAt(t *testing.T) { } func TestGetReportsTheDefaultEndAtWithoutADeadline(t *testing.T) { - sb := liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04", "tok-9") + sb := liveSandbox("e2b-aaa", "sb_one", "node-a", "registry/rt:24.04") sb.CreationTimestamp = metav1.NewTime(time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)) if got := getDetail(t, sb); got.EndAt != "2030-01-02T03:04:20Z" { t.Errorf("endAt = %q, want startedAt + %ds", got.EndAt, DefaultTimeoutSeconds) } } -// TestListReturnsArray: the SDK decodes a bare JSON array, not an object. func TestListReturnsArray(t *testing.T) { store := &fakeStore{items: []sandboxv1beta1.Sandbox{ - liveSandbox("a", "sb_one", "node-a", "i:1", "t"), - liveSandbox("b", "sb_two", "node-b", "i:2", "t"), + liveSandbox("a", "sb_one", "node-a", "i:1"), + liveSandbox("b", "sb_two", "node-b", "i:2"), }} h := newTestServer(t, store) @@ -245,8 +228,6 @@ func TestListReturnsArray(t *testing.T) { } } -// TestListEmptyIsArrayNotNull: an empty pool must encode as [], since `null` -// breaks callers that iterate the result directly. func TestListEmptyIsArrayNotNull(t *testing.T) { h := newTestServer(t, &fakeStore{}) w := do(t, h, http.MethodGet, "/sandboxes", "", testKey) @@ -255,10 +236,8 @@ func TestListEmptyIsArrayNotNull(t *testing.T) { } } -// TestTimeoutAndRefreshAckLiveSandbox: both verbs confirm the sandbox exists -// before acknowledging, so a caller never gets an OK for a dead sandbox. func TestTimeoutAndRefreshAckLiveSandbox(t *testing.T) { - store := &fakeStore{items: []sandboxv1beta1.Sandbox{liveSandbox("a", "sb_one", "node-a", "i", "t")}} + store := &fakeStore{items: []sandboxv1beta1.Sandbox{liveSandbox("a", "sb_one", "node-a", "i")}} h := newTestServer(t, store) for _, tc := range []struct{ name, path, body string }{ @@ -279,7 +258,6 @@ func TestTimeoutAndRefreshAckLiveSandbox(t *testing.T) { }) } -// TestErrorBodyCarriesMessage: the SDK surfaces `message` from the envelope. func TestErrorBodyCarriesMessage(t *testing.T) { h := newTestServer(t, &fakeStore{}) w := do(t, h, http.MethodPost, "/sandboxes", `{}`, testKey) @@ -292,9 +270,6 @@ func TestErrorBodyCarriesMessage(t *testing.T) { } } -// TestLookupUsesClaimIDResolver pins the id-keyed fast path over the real -// scatter-gather store: both id spellings resolve without a fleet List, other -// namespaces stay invisible, and a miss is a plain not-found. func TestLookupUsesClaimIDResolver(t *testing.T) { src := scale.NewStaticInventorySource() src.Put(&scale.NodeInventory{ @@ -329,14 +304,12 @@ func TestLookupUsesClaimIDResolver(t *testing.T) { } } -// TestDetailStateFollowsThePhaseLabel pins the e2b state mapping: a Hibernated -// phase reports paused, anything else running. func TestDetailStateFollowsThePhaseLabel(t *testing.T) { s, err := NewServer(&fakeStore{}, Options{AllowAnonymous: true}) if err != nil { t.Fatalf("NewServer: %v", err) } - sb := liveSandbox("sb-1", "sb_0123abcd", "node-a", "img", "tok") + sb := liveSandbox("sb-1", "sb_0123abcd", "node-a", "img") if got := s.detailFor(&sb).State; got != StateRunning { t.Fatalf("running sandbox state = %q, want %q", got, StateRunning) } @@ -346,8 +319,20 @@ func TestDetailStateFollowsThePhaseLabel(t *testing.T) { } } -// fakeStore records what the compat layer asked of the store and replays canned -// answers, so the tests assert the translation rather than the node behavior. +func TestDeleteWithoutAnOwningNodeIs500(t *testing.T) { + sb := liveSandbox("e2b-aaa", "sb_one", "node-a", "img:1") + sb.Status.NodeName = "" + store := &fakeStore{items: []sandboxv1beta1.Sandbox{sb}} + h := newTestServer(t, store) + + if w := do(t, h, http.MethodDelete, "/sandboxes/sb_one", "", testKey); w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500: a 204 would report an unreleased sandbox as freed", w.Code) + } + if store.releasedID != "" { + t.Errorf("released %q with no owning node", store.releasedID) + } +} + type fakeStore struct { claimPool scale.PoolKey claimNS string @@ -375,6 +360,18 @@ func (f *fakeStore) Get(context.Context, string, string) (*sandboxv1beta1.Sandbo return nil, nil } +func (f *fakeStore) GetByClaimID(_ context.Context, _, _ string, match func(string) bool) (*sandboxv1beta1.Sandbox, error) { + if f.listErr != nil { + return nil, f.listErr + } + for i := range f.items { + if match(f.items[i].Annotations[scale.ClaimIDAnnotation]) { + return &f.items[i], nil + } + } + return nil, k8serrors.NewNotFound(sandboxv1beta1.Resource("sandboxes"), "") +} + func (f *fakeStore) Watch(context.Context, scale.ListOptions) (watch.Interface, error) { return nil, nil } @@ -392,8 +389,6 @@ func (f *fakeStore) Release(_ context.Context, node, id string) error { return f.releaseErr } -// The lifecycle verbs are not exercised by these tests; they satisfy the -// SandboxStore contract so the fake stays a drop-in. func (f *fakeStore) Pause(context.Context, string, string) error { return nil } func (f *fakeStore) Resume(context.Context, string, string) error { return nil } @@ -410,14 +405,6 @@ func (f *fakeStore) Snapshots(context.Context, string) ([]scale.Snapshot, error) func (f *fakeStore) DeleteSnapshot(context.Context, string, string) error { return nil } -func (f *fakeStore) ClaimSnapshot(context.Context, string, string, int) (scale.Assignment, error) { - return scale.Assignment{}, nil -} - -func (f *fakeStore) Promote(context.Context, string, string, string) (scale.PoolKey, error) { - return scale.PoolKey{}, nil -} - func (f *fakeStore) Stats(context.Context, string, string) (scale.SandboxStats, error) { return scale.SandboxStats{}, nil } @@ -465,18 +452,14 @@ func do(t *testing.T, h http.Handler, method, path, body, key string) *httptest. return w } -// liveSandbox builds a sandbox as the store's scatter-gather read reports it. -func liveSandbox(name, claimID, node, image, token string) sandboxv1beta1.Sandbox { +func liveSandbox(name, claimID, node, template string) sandboxv1beta1.Sandbox { sb := sandboxv1beta1.Sandbox{ Name: name, Namespace: "sandboxes", CreationTimestamp: metav1.Now(), - Annotations: map[string]string{ - scale.ClaimIDAnnotation: claimID, - tokenAnnotation: token, - }, + Labels: map[string]string{scale.NodeLabel: node, scale.TemplateLabel: template}, + Annotations: map[string]string{scale.ClaimIDAnnotation: claimID}, } - sb.Spec.PodTemplate.Spec.Containers = []corev1.Container{{Name: "c", Image: image}} sb.Status.NodeName = node return sb } diff --git a/pkg/e2bcompat/types.go b/pkg/e2bcompat/types.go index a672f77..8509c13 100644 --- a/pkg/e2bcompat/types.go +++ b/pkg/e2bcompat/types.go @@ -1,10 +1,7 @@ package e2bcompat -// The wire types below mirror the e2b REST API schemas the e2b SDKs consume -// (e2b-dev/E2B spec/openapi.yml: NewSandbox, Sandbox, SandboxDetail, -// ListedSandbox, ResumedSandbox). Field names and JSON casing are fixed by that -// contract — the SDK unmarshals them directly — so they are reproduced exactly -// rather than restyled to this repo's own conventions. +// Field names and JSON casing below are fixed by the e2b OpenAPI contract the +// SDKs unmarshal directly, so they are reproduced rather than restyled. // Sandbox states reported to the SDK (spec: SandboxState). const ( diff --git a/pkg/podruntime/cocoon_test.go b/pkg/podruntime/cocoon_test.go index 028dd1e..b624f3c 100644 --- a/pkg/podruntime/cocoon_test.go +++ b/pkg/podruntime/cocoon_test.go @@ -57,9 +57,7 @@ func TestMutatePod(t *testing.T) { t.Fatalf("vk-cocoon mutation = %v, want %v", gotVKC, tt.wantVKC) } if !tt.wantVKC { - // Standard runtime must leave the Pod untouched: no virtual-node - // selector, no vk toleration, and none of the cocoon runtime - // annotations. This is the core --default-runtime=standard invariant. + if _, found := pod.Spec.NodeSelector[vkNodeLabelKey]; found { t.Errorf("standard runtime leaked node selector %s", vkNodeLabelKey) } diff --git a/pkg/podruntime/sandboxd_test.go b/pkg/podruntime/sandboxd_test.go index f366f94..93c7273 100644 --- a/pkg/podruntime/sandboxd_test.go +++ b/pkg/podruntime/sandboxd_test.go @@ -8,10 +8,6 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" ) -// TestMutateSandboxdRoutesToHotPool: sandboxd mode pins the pod to the -// vk-sandbox virtual node, tolerates its taint, stamps the runtime, and -// defaults the claim template from the container image — the contract the -// vk-sandbox provider consumes. vk-cocoon must NOT be involved. func TestMutateSandboxdRoutesToHotPool(t *testing.T) { m, err := NewMutator(ModeSandboxd) if err != nil { @@ -36,7 +32,7 @@ func TestMutateSandboxdRoutesToHotPool(t *testing.T) { if pod.Annotations[sandboxdTemplateAnnotation] != "base:24.04" { t.Fatalf("template default = %q, want base:24.04", pod.Annotations[sandboxdTemplateAnnotation]) } - // No cocoon MicroVM annotations leaked in. + if _, ok := pod.Annotations[cocoonModeAnnotation]; ok { t.Fatal("sandboxd pod must not carry cocoon vk-cocoon annotations") } @@ -45,7 +41,6 @@ func TestMutateSandboxdRoutesToHotPool(t *testing.T) { } } -// TestMutateSandboxdRespectsExplicitTemplate: a user-set template is preserved. func TestMutateSandboxdRespectsExplicitTemplate(t *testing.T) { m, _ := NewMutator(ModeStandard) sandbox := &sandboxv1beta1.Sandbox{Name: "sb", Namespace: "ns"} @@ -62,7 +57,6 @@ func TestMutateSandboxdRespectsExplicitTemplate(t *testing.T) { } } -// TestMutateSandboxdRejectsPinnedNode: a pinned NodeName would misroute. func TestMutateSandboxdRejectsPinnedNode(t *testing.T) { m, _ := NewMutator(ModeSandboxd) sandbox := &sandboxv1beta1.Sandbox{Name: "sb", Namespace: "ns"} @@ -73,7 +67,6 @@ func TestMutateSandboxdRejectsPinnedNode(t *testing.T) { } } -// TestNewMutatorAcceptsSandboxd guards the flag surface. func TestNewMutatorAcceptsSandboxd(t *testing.T) { if _, err := NewMutator(ModeSandboxd); err != nil { t.Fatalf("NewMutator must accept sandboxd: %v", err) diff --git a/pkg/sandboxd/client_test.go b/pkg/sandboxd/client_test.go index 1780f94..82cd56a 100644 --- a/pkg/sandboxd/client_test.go +++ b/pkg/sandboxd/client_test.go @@ -14,7 +14,6 @@ import ( func TestClaimSuccess(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // assert (not require) inside a handler goroutine: t.Errorf is goroutine-safe. assert.Equal(t, http.MethodPost, r.Method) assert.Equal(t, "/v1/claim", r.URL.Path) assert.Equal(t, "Bearer root-token", r.Header.Get("Authorization")) @@ -38,8 +37,6 @@ func TestClaimSuccess(t *testing.T) { require.Equal(t, "10.0.0.5:7777", res.OwnerAddr) } -// TestSandboxdClaimFallbackOn429 asserts a 429 maps to ErrNodeAtCapacity, the -// signal the gateway turns into an L1 fallback. func TestSandboxdClaimFallbackOn429(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTooManyRequests) @@ -52,8 +49,6 @@ func TestSandboxdClaimFallbackOn429(t *testing.T) { require.ErrorIs(t, err, ErrNodeAtCapacity) } -// TestSandboxdClaimRedirectIsCapacityMiss asserts a 200 that carries only a peer -// redirect (no delivered id) is treated as a capacity miss, not a bogus success. func TestSandboxdClaimRedirectIsCapacityMiss(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -89,16 +84,16 @@ func TestReleaseSuccessAndAlreadyGone(t *testing.T) { assert.Equal(t, "Bearer sbtok", r.Header.Get("Authorization"), "release authenticates with the sandbox's own token") assert.Equal(t, "/v1/sandboxes/sb_abc/release", r.URL.Path) if n == 1 { - w.WriteHeader(http.StatusNoContent) // first: destroyed + w.WriteHeader(http.StatusNoContent) } else { - w.WriteHeader(http.StatusNotFound) // second: already gone + w.WriteHeader(http.StatusNotFound) } })) defer srv.Close() c := New(srv.URL, "root-token") require.NoError(t, c.Release(t.Context(), "sb_abc", "sbtok")) - // 404 (already gone) is treated as success. + require.NoError(t, c.Release(t.Context(), "sb_abc", "sbtok")) require.Equal(t, int64(2), releases.Load()) } diff --git a/pkg/sandboxd/lifecycle.go b/pkg/sandboxd/lifecycle.go index 6d5f984..bc38c82 100644 --- a/pkg/sandboxd/lifecycle.go +++ b/pkg/sandboxd/lifecycle.go @@ -11,12 +11,14 @@ import ( "time" ) -// The lifecycle verbs below all authenticate with the node's root api_token -// (the fleet token this client already holds) and take sandboxd's operator -// path, which resolves the sandbox by id without a per-sandbox token. That is -// deliberate: keeping one secret per sandbox in the control plane would turn -// its O(nodes) storage into O(sandboxes), the property the whole design rests -// on. sandboxd authorizes these by the root token exactly as it does release. +// maxReplyBytes caps a sandboxd reply body. At the measured 213 bytes per +// listed sandbox it clears 78k sandboxes on one node, well past the 2000 the +// design projects. +const maxReplyBytes = 16 << 20 + +// The lifecycle verbs take sandboxd's operator path under the fleet root token: +// a per-sandbox secret would turn the control plane's O(nodes) storage into +// O(sandboxes). // ForkSpec is the POST /v1/sandboxes/{id}/fork body. Token stays empty on the // operator path; Count must be within the node's max_fork_count. @@ -193,17 +195,6 @@ func (c *Client) Promote(ctx context.Context, id string, spec PromoteSpec) (Pool return out.Key, err } -// Sandbox performs GET /v1/sandboxes/{id}, the single-sandbox read that avoids -// scanning the whole-node listing. -func (c *Client) Sandbox(ctx context.Context, id string) (SandboxSummary, error) { - var out SandboxSummary - if id == "" { - return out, fmt.Errorf("sandboxd: sandbox read requires an id") - } - err := c.getJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id), &out) - return out, err -} - // Stats performs GET /v1/sandboxes/{id}/stats. func (c *Client) Stats(ctx context.Context, id string) (SandboxStats, error) { var out SandboxStats @@ -301,12 +292,18 @@ func (c *Client) getJSON(ctx context.Context, path string, out any) error { return decodeInto(resp, out, path) } -// decodeInto reads a bounded reply body into out. +// decodeInto reads a bounded reply body into out. Reading one byte past the cap +// distinguishes an over-cap reply from malformed JSON: silently truncating a +// node's sandbox or checkpoint listing would surface as "unexpected end of JSON +// input" and make that node's sandboxes vanish from the aggregated view. func decodeInto(resp *http.Response, out any, path string) error { - b, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + b, err := io.ReadAll(io.LimitReader(resp.Body, maxReplyBytes+1)) if err != nil { return fmt.Errorf("sandboxd: read %s reply: %w", path, err) } + if len(b) > maxReplyBytes { + return fmt.Errorf("sandboxd: %s reply exceeds %d bytes", path, maxReplyBytes) + } if err := json.Unmarshal(b, out); err != nil { return fmt.Errorf("sandboxd: decode %s reply: %w", path, err) } diff --git a/pkg/scale/apiserver/lifecycle_storage.go b/pkg/scale/apiserver/lifecycle_storage.go index 461f88b..1390c87 100644 --- a/pkg/scale/apiserver/lifecycle_storage.go +++ b/pkg/scale/apiserver/lifecycle_storage.go @@ -15,12 +15,9 @@ import ( "github.com/cocoonstack/sandbox-operator/pkg/scale" ) -// The lifecycle subresources are POST-only action verbs, the pods/eviction -// shape rather than the pods/status one: each is a synchronous node-local -// transaction against an already-delivered sandbox, so there is nothing to GET -// and nothing to reconcile. Keeping them as subresources leaves the standard -// Sandbox schema untouched, which is what lets an unmodified upstream -// agent-sandbox client keep working against this server. +// The lifecycle subresources take the pods/eviction shape, not pods/status: +// each is a synchronous node-local transaction with nothing to GET, and keeping +// them as subresources leaves the standard Sandbox schema untouched. var ( _ rest.Storage = &lifecycleREST{} diff --git a/pkg/scale/apiserver/openapi_test.go b/pkg/scale/apiserver/openapi_test.go index 35e8715..0d87b0d 100644 --- a/pkg/scale/apiserver/openapi_test.go +++ b/pkg/scale/apiserver/openapi_test.go @@ -13,12 +13,6 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" ) -// TestManagedFieldsTypeConverterResolvesSandbox reproduces the create-path crux -// that produced "[SHOULD NOT HAPPEN] failed to update managedFields" on every -// write: the managed-fields TypeConverter must map the Sandbox GVK to a model -// (via the x-kubernetes-group-version-kind marker), else ObjectToTyped returns -// NoCorrespondingTypeError. With the old empty OpenAPI this failed; with -// sandboxOpenAPIDefinitions it must succeed for both Sandbox and SandboxList. func TestManagedFieldsTypeConverterResolvesSandbox(t *testing.T) { cfg := NewOpenAPIV3Config() models, err := builder3.BuildOpenAPIDefinitionsForResources(cfg, @@ -48,17 +42,6 @@ func TestManagedFieldsTypeConverterResolvesSandbox(t *testing.T) { } } -// TestEveryServedTypeHasAnOpenAPIModel reproduces a real deployment failure: -// the action subresources were registered in the Scheme but had no OpenAPI -// model, and InstallAPIGroup then refused to start the server outright with -// -// unable to get openapi models: cannot find model definition for -// io.k8s.apimachinery.pkg.apis.meta.v1.TypeMeta -// -// because the subresource bodies embed metav1.TypeMeta and openapinamer -// resolves them through this map, not the Scheme. A missing model is not a -// degraded feature — it is a crash loop on rollout, so every served kind must -// be present here. func TestEveryServedTypeHasAnOpenAPIModel(t *testing.T) { defs := sandboxOpenAPIDefinitions(func(path string) spec.Ref { return spec.Ref{} }) @@ -95,10 +78,6 @@ func TestEveryServedTypeHasAnOpenAPIModel(t *testing.T) { } } -// TestInstallSandboxAPISucceeds is the production startup path: main.go builds -// a GenericAPIServer and calls InstallSandboxAPI. An incomplete OpenAPI model -// graph fails HERE, and the binary then crash-loops on rollout — so this must -// be exercised locally, not discovered in the cluster. func TestInstallSandboxAPISucceeds(t *testing.T) { cfg := genericapiserver.NewConfig(Codecs) cfg.EffectiveVersion = apiservercompatibility.DefaultBuildEffectiveVersion() diff --git a/pkg/scale/apiserver/storage.go b/pkg/scale/apiserver/storage.go index ef8f5d9..84b99d8 100644 --- a/pkg/scale/apiserver/storage.go +++ b/pkg/scale/apiserver/storage.go @@ -38,9 +38,9 @@ const ( AddressAnnotation = "sandbox.cocoonstack.io/address" // TokenAnnotation carries the per-sandbox ownership token so a caller can // exec/agent into the sandbox it claimed via the L3 apiserver. - TokenAnnotation = "sandbox.cocoonstack.io/token" + TokenAnnotation = scale.TokenAnnotation // NetAnnotation selects the pool network mode on Create (default "none"). - NetAnnotation = "sandbox.cocoonstack.io/net" + NetAnnotation = scale.NetAnnotation // TTLSecondsAnnotation bounds the claim lease in whole seconds on Create for // clients that cannot set spec.shutdownTime (0 = the owning node's default). TTLSecondsAnnotation = "sandbox.cocoonstack.io/ttl-seconds" @@ -169,8 +169,11 @@ func (r *sandboxREST) Delete(ctx context.Context, name string, deleteValidation node := sb.Status.NodeName if node == "" { - // No owning node resolved: nothing to release against. Report it deleted. - return sb, true, nil + // The inventory entry exists but names no node, so the release cannot be + // routed. Reporting success here would leak the microVM to its TTL. + return nil, false, apierrors.NewInternalError(fmt.Errorf( + "cannot delete sandbox %s/%s: inventory entry names no owning node; refusing to report it released", + namespace, name)) } claimID := sb.Annotations[ClaimIDAnnotation] if claimID == "" { @@ -209,10 +212,12 @@ func toScaleListOptions(ctx context.Context, options *metainternalversion.ListOp // poolKeyForSandbox derives the warm-pool key from a Sandbox: the template is the // first container's image, the size is a t-shirt class mapped from that container's -// resources, and the net comes from the NetAnnotation (default "none"). It defers -// to scale.PoolKeyFor so the SandboxWarmPool driver derives an identical key. +// resources, and the net comes from the NetAnnotation on the object or its pod +// template (default "none"). It defers to scale.PoolKeyFor and +// scale.NetForAnnotations so the SandboxWarmPool driver derives an identical key. func poolKeyForSandbox(sb *sandboxv1beta1.Sandbox) scale.PoolKey { - return scale.PoolKeyFor(sb.Spec.PodTemplate.Spec.Containers, sb.Annotations[NetAnnotation]) + net := scale.NetForAnnotations(sb.Annotations, sb.Spec.PodTemplate.ObjectMeta.Annotations) + return scale.PoolKeyFor(sb.Spec.PodTemplate.Spec.Containers, net) } // ttlSecondsForSandbox derives the claim lease: spec.shutdownTime wins, the diff --git a/pkg/scale/apiserver/storage_test.go b/pkg/scale/apiserver/storage_test.go index b903237..cb796f9 100644 --- a/pkg/scale/apiserver/storage_test.go +++ b/pkg/scale/apiserver/storage_test.go @@ -17,9 +17,6 @@ import ( "github.com/cocoonstack/sandbox-operator/pkg/scale" ) -// TestDelete_ReleasesByClaimIDAnnotation proves Delete releases the microVM by -// the sandboxd claim id the node published (the claim-id annotation), never by -// the k8s object name. func TestDelete_ReleasesByClaimIDAnnotation(t *testing.T) { sb := &sandboxv1beta1.Sandbox{ Namespace: "ns", @@ -39,9 +36,6 @@ func TestDelete_ReleasesByClaimIDAnnotation(t *testing.T) { assert.Equal(t, "sb_abc123", store.releaseID, "must release by the sandboxd claim id, not the k8s name") } -// TestDelete_FailsLoudWithoutClaimID proves Delete refuses to release when the -// node has not published a claim id — releasing by the k8s name (the old bug) -// would target the wrong claim. It must error and release nothing. func TestDelete_FailsLoudWithoutClaimID(t *testing.T) { sb := &sandboxv1beta1.Sandbox{ Namespace: "ns", Name: "s1", @@ -126,8 +120,6 @@ func TestCreate_ReportsGrantedDeadline(t *testing.T) { assert.Nil(t, out.Spec.ShutdownTime, "the submitted spec is echoed, not rewritten") } -// fakeStore is a scale.SandboxStore stub: Get returns a preset sandbox, Claim -// and Release record their arguments. The read-path verbs are unused. type fakeStore struct { getSandbox *sandboxv1beta1.Sandbox getErr error @@ -165,8 +157,6 @@ func (f *fakeStore) Release(_ context.Context, node, id string) error { return f.releaseErr } -// The lifecycle verbs are not exercised by these tests; they satisfy the -// SandboxStore contract so the fake stays a drop-in. func (f *fakeStore) Pause(context.Context, string, string) error { return nil } func (f *fakeStore) Resume(context.Context, string, string) error { return nil } diff --git a/pkg/scale/claimgateway_impl.go b/pkg/scale/claimgateway_impl.go index 7497c0f..bf794b6 100644 --- a/pkg/scale/claimgateway_impl.go +++ b/pkg/scale/claimgateway_impl.go @@ -61,8 +61,6 @@ type SandboxdClient interface { Checkpoint(ctx context.Context, id string, spec sandboxd.CheckpointSpec) (sandboxd.Checkpoint, error) Checkpoints(ctx context.Context) ([]sandboxd.Checkpoint, error) DeleteCheckpoint(ctx context.Context, checkpointID string) error - ClaimCheckpoint(ctx context.Context, checkpointID string, spec sandboxd.CheckpointClaimSpec) (sandboxd.ClaimResult, error) - Promote(ctx context.Context, id string, spec sandboxd.PromoteSpec) (sandboxd.PoolKey, error) Stats(ctx context.Context, id string) (sandboxd.SandboxStats, error) } diff --git a/pkg/scale/claimgateway_impl_test.go b/pkg/scale/claimgateway_impl_test.go index a2c106b..61c5119 100644 --- a/pkg/scale/claimgateway_impl_test.go +++ b/pkg/scale/claimgateway_impl_test.go @@ -25,10 +25,6 @@ import ( "github.com/cocoonstack/sandbox-operator/pkg/sandboxd" ) -// TestClaimGatewayHappyPath: a warm sandbox is delivered and the Assignment is -// returned immediately; the SandboxClaim is recorded Bound asynchronously (the -// node acts first, the apiserver records after). No VM is destroyed on the claim -// path. func TestClaimGatewayHappyPath(t *testing.T) { fs := newFakeSandboxd(t) fc := newClaimClient(t, "c1") @@ -49,7 +45,6 @@ func TestClaimGatewayHappyPath(t *testing.T) { require.Equal(t, "node-a", a.Node) require.Equal(t, "10.0.0.5:7777", a.Address) - // The record follows the action: wait for the async job, then assert Bound. gw.Wait() cur := getClaim(t, fc, "c1") require.Equal(t, a.SandboxName, cur.Status.SandboxStatus.Name, "async RecordBound should have set status.sandbox.name") @@ -60,15 +55,10 @@ func TestClaimGatewayHappyPath(t *testing.T) { require.Equal(t, int64(0), fs.releases.Load(), "claim path must never destroy a VM") } -// TestClaimGatewayOrphanBindingConverges: the async Bound record is lost (gateway -// crash), leaving an orphan binding. The OrphanReconciler adopts it — records the -// missing Bound and converges the orphan count to 0 — and crucially destroys NO -// VM (the sandboxd release endpoint is never hit). func TestClaimGatewayOrphanBindingConverges(t *testing.T) { fs := newFakeSandboxd(t) fc := newClaimClient(t, "c-orphan") - // Gateway whose async record always fails → the delivery is never recorded. gw := NewGateway(GatewayConfig{ Node: "node-a", Client: fs.client(), Authorizer: allowAuthorizer{}, Recorder: failingRecorder{}, BaseContext: t.Context(), Logger: testr.New(t), @@ -77,7 +67,6 @@ func TestClaimGatewayOrphanBindingConverges(t *testing.T) { require.NoError(t, err) gw.Wait() - // Orphan confirmed: delivered, but the claim carries no Bound record. require.Empty(t, getClaim(t, fc, "c-orphan").Status.SandboxStatus.Name) inv := sliceInventory{{SandboxName: a.SandboxName, Node: a.Node, Address: a.Address, ClaimNS: "default", ClaimName: "c-orphan"}} @@ -88,7 +77,6 @@ func TestClaimGatewayOrphanBindingConverges(t *testing.T) { require.Equal(t, 1, n, "the single orphan binding should be reconciled") require.Equal(t, a.SandboxName, getClaim(t, fc, "c-orphan").Status.SandboxStatus.Name, "orphan binding converged to Bound") - // Idempotent: a second pass finds no orphans. n, err = orc.Reconcile(t.Context()) require.NoError(t, err) require.Equal(t, 0, n, "orphan count must converge to 0") @@ -96,9 +84,6 @@ func TestClaimGatewayOrphanBindingConverges(t *testing.T) { require.Equal(t, int64(0), fs.releases.Load(), "orphan GC must NEVER destroy a VM") } -// TestClaimGatewayFallbackOnNoCapacity: when sandboxd reports no warm capacity -// (429), Claim returns an error for which IsFallback is true, so the caller drops -// to the L1 Kubernetes path. func TestClaimGatewayFallbackOnNoCapacity(t *testing.T) { fs := newFakeSandboxd(t) fs.forceStatus.Store(http.StatusTooManyRequests) @@ -117,12 +102,6 @@ func TestClaimGatewayFallbackOnNoCapacity(t *testing.T) { require.Equal(t, int64(0), fs.releases.Load()) } -// TestClaimGatewayReleaseOwnerTeardownOnly: Release destroys the VM ONLY for a -// sandbox the gateway actually delivered (owner-authorized teardown holds the -// sandbox's own token, which the gateway keeps). An Assignment the gateway never -// handed out — e.g. one synthesized from stale pod state — is refused with NO -// sandboxd call, so pod-level state can never drive a destroy. This encodes the -// G-0131 delete-authorization contract. func TestClaimGatewayReleaseOwnerTeardownOnly(t *testing.T) { fs := newFakeSandboxd(t) @@ -135,23 +114,18 @@ func TestClaimGatewayReleaseOwnerTeardownOnly(t *testing.T) { require.NoError(t, err) gw.Wait() - // Owner-authorized teardown of a delivered sandbox destroys exactly that VM. require.NoError(t, gw.Release(t.Context(), a)) require.Equal(t, int64(1), fs.releases.Load(), "owner teardown must destroy the delivered VM") - // An Assignment the gateway never delivered (pod-derived / stale) must NOT - // reach sandboxd — no code path from pod state to a VM destroy. err = gw.Release(t.Context(), Assignment{SandboxName: "sb_never_delivered", Node: "node-a"}) require.Error(t, err) require.Equal(t, int64(1), fs.releases.Load(), "an undelivered Assignment must not trigger any destroy") - // A second release of the same sandbox is likewise refused (already handed back). err = gw.Release(t.Context(), a) require.Error(t, err) require.Equal(t, int64(1), fs.releases.Load()) } -// TestClaimGatewayAuthorizationRejectsInline verifies SAR denials precede delivery. func TestClaimGatewayAuthorizationRejectsInline(t *testing.T) { t.Run("deny", func(t *testing.T) { fs := newFakeSandboxd(t) @@ -203,15 +177,12 @@ func TestClaimGatewayAuthorizationRejectsInline(t *testing.T) { }) } -// fakeSandboxd is an httptest-backed sandboxd. It serves POST /v1/claim (200 with -// a fresh id, unless forceStatus overrides) and POST /v1/sandboxes/{id}/release -// (204), counting both so tests can assert the VM-destroy path. type fakeSandboxd struct { srv *httptest.Server claims atomic.Int64 releases atomic.Int64 nextID atomic.Int64 - forceStatus atomic.Int64 // when non-zero, claim returns this status (e.g. 429) + forceStatus atomic.Int64 } func newFakeSandboxd(t *testing.T) *fakeSandboxd { @@ -256,16 +227,12 @@ type noopRecorder struct{} func (noopRecorder) RecordBound(context.Context, string, string, Assignment) error { return nil } -// failingRecorder always fails — modeling the async Bound record lost to a gateway -// crash, which leaves an orphan binding for the OrphanReconciler to heal. type failingRecorder struct{} func (failingRecorder) RecordBound(context.Context, string, string, Assignment) error { return fmt.Errorf("simulated record failure (gateway crashed before recording Bound)") } -// sliceInventory is a NodeInventorySource backed by a fixed slice (as if read from -// sandboxd's own inventory after a crash). type sliceInventory []Delivery func (s sliceInventory) LiveDeliveries(context.Context) ([]Delivery, error) { diff --git a/pkg/scale/nodeindex.go b/pkg/scale/nodeindex.go new file mode 100644 index 0000000..daae7d4 --- /dev/null +++ b/pkg/scale/nodeindex.go @@ -0,0 +1,61 @@ +package scale + +import "sync" + +// nodeIndexMaxEntries bounds one generation of the resolution index. +const nodeIndexMaxEntries = 8192 + +// nodeIndex maps a lookup key to the node last seen holding it, so a repeat +// resolve reads one node's inventory instead of sweeping the fleet. +type nodeIndex struct { + mu sync.Mutex + max int + cur map[string]string + prev map[string]string +} + +func newNodeIndex(maxEntries int) *nodeIndex { + return &nodeIndex{max: maxEntries, cur: map[string]string{}, prev: map[string]string{}} +} + +func (x *nodeIndex) lookup(key string) (string, bool) { + x.mu.Lock() + defer x.mu.Unlock() + if node, ok := x.cur[key]; ok { + return node, true + } + node, ok := x.prev[key] + if ok { + x.storeLocked(key, node) + } + return node, ok +} + +func (x *nodeIndex) remember(key, node string) { + if key == "" || node == "" { + return + } + x.mu.Lock() + defer x.mu.Unlock() + x.storeLocked(key, node) +} + +func (x *nodeIndex) forget(key string) { + x.mu.Lock() + defer x.mu.Unlock() + delete(x.cur, key) + delete(x.prev, key) +} + +// storeLocked retires the whole generation once it fills, which bounds the +// index at 2*max keys without per-entry recency bookkeeping. +func (x *nodeIndex) storeLocked(key, node string) { + if len(x.cur) >= x.max { + x.prev, x.cur = x.cur, make(map[string]string, x.max/2) + } + x.cur[key] = node +} + +func nameKey(namespace, name string) string { return "name/" + namespace + "/" + name } + +func claimKey(namespace, id string) string { return "claim/" + namespace + "/" + id } diff --git a/pkg/scale/poolkey.go b/pkg/scale/poolkey.go index 10426e2..970c3fb 100644 --- a/pkg/scale/poolkey.go +++ b/pkg/scale/poolkey.go @@ -1,6 +1,8 @@ package scale import ( + "cmp" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -43,6 +45,13 @@ func PoolKeyFor(containers []corev1.Container, net string) PoolKey { return PoolKey{Template: template, Net: net, Size: SizeClassForContainers(containers)} } +// NetForAnnotations resolves the pool network axis from an object's own +// annotations, falling back to its pod template's. Create and the warm-pool +// driver resolve it through this one function so both derive the same key. +func NetForAnnotations(object, podTemplate map[string]string) string { + return cmp.Or(object[NetAnnotation], podTemplate[NetAnnotation]) +} + // SizeClassForContainers maps the first container's CPU/memory onto small|medium| // large. It prefers requests, falls back to limits, and defaults to "small" when // neither is set. Thresholds: >4 CPU or >8Gi -> large; >1 CPU or >2Gi -> medium. diff --git a/pkg/scale/sandboxstore.go b/pkg/scale/sandboxstore.go index ef0d52e..800d4bc 100644 --- a/pkg/scale/sandboxstore.go +++ b/pkg/scale/sandboxstore.go @@ -95,15 +95,18 @@ type SandboxLifecycle interface { Snapshots(ctx context.Context, node string) ([]Snapshot, error) // DeleteSnapshot removes a checkpoint. A missing checkpoint is success. DeleteSnapshot(ctx context.Context, node, snapshotID string) error - // ClaimSnapshot delivers a fresh sandbox branched from a checkpoint. - ClaimSnapshot(ctx context.Context, node, snapshotID string, ttlSeconds int) (Assignment, error) - // Promote publishes the sandbox as a node-local template that later claims - // for that key clone from. - Promote(ctx context.Context, node, id, template string) (PoolKey, error) // Stats reports one sandbox's resource usage. Stats(ctx context.Context, node, id string) (SandboxStats, error) } +// ClaimIDResolver is the store fast path that resolves one sandbox by its +// node-local claim id without materializing the fleet. id is the caller's +// spelling of the id and keys the owning-node index; match decides which +// node-local id it accepts. An empty namespace matches every namespace. +type ClaimIDResolver interface { + GetByClaimID(ctx context.Context, namespace, id string, match func(claimID string) bool) (*sandboxv1beta1.Sandbox, error) +} + // Snapshot is a captured sandbox state that new sandboxes can branch from. type Snapshot struct { ID string diff --git a/pkg/scale/sandboxstore_bench_test.go b/pkg/scale/sandboxstore_bench_test.go index e9a0812..d3f5d03 100644 --- a/pkg/scale/sandboxstore_bench_test.go +++ b/pkg/scale/sandboxstore_bench_test.go @@ -7,7 +7,6 @@ import ( extv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" ) -// Fleet shapes: the deployed 26-node fleet and the 50k-microVM projection. var benchFleets = []struct { name string nodes int @@ -17,8 +16,6 @@ var benchFleets = []struct { {"200x2000", 200, 2000}, } -// BenchmarkStoreGet resolves one sandbox living on the last node in enumeration -// order — the sequential worst case a Get pays on a miss-heavy sweep. func BenchmarkStoreGet(b *testing.B) { for _, fleet := range benchFleets { b.Run(fleet.name, func(b *testing.B) { @@ -37,8 +34,6 @@ func BenchmarkStoreGet(b *testing.B) { } } -// BenchmarkStoreWarmCandidates sweeps every node inventory for warm capacity — -// the node-pick cost every aggregated Create/claim pays. func BenchmarkStoreWarmCandidates(b *testing.B) { for _, fleet := range benchFleets { b.Run(fleet.name, func(b *testing.B) { @@ -58,8 +53,6 @@ func BenchmarkStoreWarmCandidates(b *testing.B) { } } -// benchStore publishes nodes×perNode inventory entries, every node advertising -// warm capacity for the returned pool key. func benchStore(b *testing.B, nodes, perNode int) (*scatterGatherStore, PoolKey) { b.Helper() pool := PoolKey{Template: "ghcr.io/cocoonstack/sandbox/rt:24.04", Net: NetDefault, Size: SizeClassSmall} diff --git a/pkg/scale/sandboxstore_claim_test.go b/pkg/scale/sandboxstore_claim_test.go index c708226..1819676 100644 --- a/pkg/scale/sandboxstore_claim_test.go +++ b/pkg/scale/sandboxstore_claim_test.go @@ -24,21 +24,20 @@ func TestStoreClaim_RoutesToAWarmNode(t *testing.T) { assert.Equal(t, "n2", a.Node) assert.Equal(t, "sb-abc", a.SandboxName) assert.Equal(t, "10.0.0.2:9000", a.Address) - // The claim routed to the picked node's advertise address with the uniform token. + assert.Equal(t, "10.0.0.2:7777", f.builtAddr) assert.Equal(t, "uniform-token", f.builtToken) assert.Equal(t, "img", f.claimSpec.Template) assert.Equal(t, 600, f.claimSpec.TTLSeconds, "the caller's TTL must reach sandboxd") assert.Equal(t, deadline, a.Deadline, "the node-granted deadline must ride the assignment back") - // The claim carries the k8s "/" so the node echoes it into - // its operator index and the aggregated read path can resolve this sandbox. + assert.Equal(t, "ns/s1", f.claimSpec.ClaimRef) assert.Equal(t, 1, f.claimCalls) } func TestStoreClaim_NoWarmCapacityIsRetryable(t *testing.T) { src := NewStaticInventorySource() - // Warm==0 everywhere: no node can hand over a microVM. + src.Put(poolInv("n1", "10.0.0.1:7777", PoolCapacity{Template: "img", Warm: 0, Target: 5})) f := &recordingFactory{} store := NewScatterGatherStore(src, WithLogger(logr.Discard()), WithClaimRouting("t", f.factory())) @@ -51,7 +50,7 @@ func TestStoreClaim_NoWarmCapacityIsRetryable(t *testing.T) { func TestStoreClaim_PoolKeyMatchingNormalizesDefaults(t *testing.T) { src := NewStaticInventorySource() - // A pool advertised with unset net/size serves the default-named ("none"/"small") key. + src.Put(poolInv("n1", "10.0.0.1:7777", PoolCapacity{Template: "img", Warm: 2, Target: 2})) f := &recordingFactory{claimResult: sandboxd.ClaimResult{ID: "sb-1"}} store := NewScatterGatherStore(src, WithLogger(logr.Discard()), WithClaimRouting("t", f.factory())) @@ -59,7 +58,6 @@ func TestStoreClaim_PoolKeyMatchingNormalizesDefaults(t *testing.T) { _, err := store.Claim(t.Context(), "ns", "s1", PoolKey{Template: "img", Net: "none", Size: "small"}, 0) require.NoError(t, err) - // A different net finds no matching pool. _, err = store.Claim(t.Context(), "ns", "s2", PoolKey{Template: "img", Net: "egress"}, 0) require.Error(t, err) assert.True(t, IsNoWarmCapacity(err), "net mismatch must be no-capacity, got %v", err) @@ -68,7 +66,7 @@ func TestStoreClaim_PoolKeyMatchingNormalizesDefaults(t *testing.T) { func TestStoreClaim_SandboxdCapacityRaceIsRetryable(t *testing.T) { src := NewStaticInventorySource() src.Put(poolInv("n1", "10.0.0.1:7777", PoolCapacity{Template: "img", Warm: 1, Target: 5})) - // The advertised warm count raced to zero: sandboxd answers at-capacity. + f := &recordingFactory{claimErr: sandboxd.ErrNodeAtCapacity} store := NewScatterGatherStore(src, WithLogger(logr.Discard()), WithClaimRouting("t", f.factory())) @@ -94,7 +92,7 @@ func TestStoreRelease_RoutesToNodeAddressWithUniformToken(t *testing.T) { func TestStoreClaimRelease_FailClosedWithoutRouting(t *testing.T) { src := NewStaticInventorySource() src.Put(poolInv("n1", "10.0.0.1:7777", PoolCapacity{Template: "img", Warm: 1, Target: 1})) - store := NewScatterGatherStore(src, WithLogger(logr.Discard())) // no WithClaimRouting + store := NewScatterGatherStore(src, WithLogger(logr.Discard())) _, err := store.Claim(t.Context(), "ns", "s1", PoolKey{Template: "img"}, 0) require.Error(t, err) @@ -120,8 +118,7 @@ func TestPickWarmNodeSpreadsAcrossTheFleet(t *testing.T) { best, _ := pickPowerOfTwo(candidates) picked[best.node]++ } - // Deterministic max-first would put all 200 on one node. Equal warmth means - // power-of-two sampling must reach every node. + assert.Len(t, picked, 4, "burst funneled onto a subset: %v", picked) } @@ -140,15 +137,11 @@ func TestPickWarmNodePrefersTheWarmerSample(t *testing.T) { warmPicks++ } } - // Two samples with replacement pick the cold node only when both land on it. + assert.Greater(t, warmPicks, 130, "sampling lost its bias toward warm capacity") } func TestStoreClaimFallsBackWhenTheSampledNodeRacedToZero(t *testing.T) { - // The classic stale-inventory shape: a node still advertising one warm - // microVM it no longer has, next to a node that is genuinely warm. Sampling - // picks the empty one often enough that giving up on it would 503 callers - // the fleet can serve. src := NewStaticInventorySource() src.Put(poolInv("stale", "stale:7777", PoolCapacity{Template: "img", Warm: 1, Target: 5})) src.Put(poolInv("warm", "warm:7777", PoolCapacity{Template: "img", Warm: 100, Target: 200})) @@ -177,8 +170,6 @@ func TestStoreClaimReportsNoCapacityOnlyWhenEveryNodeRaced(t *testing.T) { assert.Equal(t, 2, f.calls, "each node must be tried exactly once") } -// raceFactory answers ErrNodeAtCapacity for nodes that advertised warm capacity -// they no longer hold. type raceFactory struct { emptyAddr string emptyAll bool @@ -192,8 +183,6 @@ func (r *raceFactory) factory() SandboxdClientFactory { } } -// raceClient embeds recordingClient for the rest of the SandboxdClient surface -// and only decides whether this node still has the warm microVM it advertised. type raceClient struct { *recordingClient f *raceFactory @@ -208,7 +197,6 @@ func (c *raceClient) Claim(context.Context, sandboxd.ClaimSpec) (sandboxd.ClaimR return c.f.result, nil } -// poolInv builds a NodeInventory advertising a sandboxd address and pool capacities. func poolInv(node, addr string, pools ...PoolCapacity) *NodeInventory { return &NodeInventory{ Name: node, @@ -218,9 +206,6 @@ func poolInv(node, addr string, pools ...PoolCapacity) *NodeInventory { } } -// recordingFactory captures how the store built and called the per-node sandboxd -// client, so tests can assert claim/release routing (address + uniform token) and -// the derived claim spec without a live node. type recordingFactory struct { builtAddr string builtToken string @@ -261,8 +246,6 @@ func (c *recordingClient) Release(_ context.Context, id, token string) error { return c.f.releaseErr } -// The lifecycle verbs are not exercised here; they satisfy the SandboxdClient -// port so the recorder stays a drop-in. func (c *recordingClient) Hibernate(context.Context, string) error { return nil } func (c *recordingClient) Wake(context.Context, string) error { return nil } @@ -281,14 +264,6 @@ func (c *recordingClient) Checkpoints(context.Context) ([]sandboxd.Checkpoint, e func (c *recordingClient) DeleteCheckpoint(context.Context, string) error { return nil } -func (c *recordingClient) ClaimCheckpoint(context.Context, string, sandboxd.CheckpointClaimSpec) (sandboxd.ClaimResult, error) { - return sandboxd.ClaimResult{}, nil -} - -func (c *recordingClient) Promote(context.Context, string, sandboxd.PromoteSpec) (sandboxd.PoolKey, error) { - return sandboxd.PoolKey{}, nil -} - func (c *recordingClient) Stats(context.Context, string) (sandboxd.SandboxStats, error) { return sandboxd.SandboxStats{}, nil } diff --git a/pkg/scale/sandboxstore_impl.go b/pkg/scale/sandboxstore_impl.go index 69c83fd..53b4752 100644 --- a/pkg/scale/sandboxstore_impl.go +++ b/pkg/scale/sandboxstore_impl.go @@ -44,6 +44,10 @@ const ( PhaseLabel = "sandbox.cocoonstack.io/phase" // ClaimLabel carries the claim name a synthesized Sandbox is bound to. ClaimLabel = "sandbox.cocoonstack.io/claim" + // TemplateLabel carries the pool template a synthesized Sandbox was claimed + // from; it is the only recoverable source, since no per-sandbox object holds + // the pod spec the template would otherwise be read off. + TemplateLabel = "sandbox.cocoonstack.io/template" // ClaimIDAnnotation carries the owning node's sandboxd claim id ("sb_...") on a // synthesized Sandbox. Unlike the label keys above it is an annotation — an @@ -56,6 +60,12 @@ const ( // Sandbox: stamped from inventory on reads and from the claim on Create. // apiserver.DeadlineAnnotation aliases it. DeadlineAnnotation = "sandbox.cocoonstack.io/deadline" + // NetAnnotation selects the pool network mode. Create and the warm-pool + // driver must read the same key or a claim never matches provisioned warm + // capacity (perpetual 503). + NetAnnotation = "sandbox.cocoonstack.io/net" + // TokenAnnotation carries the per-sandbox ownership token handed back on Create. + TokenAnnotation = "sandbox.cocoonstack.io/token" // Connection pooling for the node-local claim path. Idle conns per host are // sized to the per-node claim fan-out so a burst reuses connections instead @@ -101,6 +111,9 @@ type InventorySource interface { // NodeInventory returns one node's authoritative inventory. A partitioned or // not-yet-published node returns an error, which List logs and skips. NodeInventory(ctx context.Context, node string) (*NodeInventory, error) + // NodeCapacity returns one node's advertise address and warm pools without + // decoding its entry list, which the claim and routing paths never read. + NodeCapacity(ctx context.Context, node string) (address string, pools []PoolCapacity, err error) } // warmCandidate is one node advertising warm capacity for a requested pool. @@ -140,14 +153,12 @@ func WithClaimRouting(token string, factory SandboxdClientFactory) StoreOption { } } -// NewSandboxdClientFactory returns the production SandboxdClientFactory: an HTTP -// sandboxd client per node advertise address (a bare "host:port" is given the -// http scheme; an address that already carries a scheme is used verbatim). -func NewSandboxdClientFactory() SandboxdClientFactory { - // One client for the whole fleet: a per-call client would fall back to - // http.DefaultTransport, whose MaxIdleConnsPerHost of 2 forces a fresh TCP - // handshake on every concurrent claim past the second to the same node. - hc := &http.Client{ +// NewSandboxdHTTPClient returns one HTTP client for the whole fleet. A per-call +// client would fall back to http.DefaultTransport, whose MaxIdleConnsPerHost of +// 2 forces a fresh TCP handshake on every concurrent claim past the second to +// the same node. +func NewSandboxdHTTPClient() *http.Client { + return &http.Client{ Timeout: sandboxdRequestTimeout, Transport: &http.Transport{ MaxIdleConns: sandboxdMaxIdleConns, @@ -155,12 +166,24 @@ func NewSandboxdClientFactory() SandboxdClientFactory { IdleConnTimeout: sandboxdIdleConnTimeout, }, } +} + +// SandboxdBaseURL renders a node advertise address as a sandboxd base URL: a +// bare "host:port" is given the http scheme, an address that already carries +// one is used verbatim. +func SandboxdBaseURL(addr string) string { + if strings.Contains(addr, "://") { + return addr + } + return "http://" + addr +} + +// NewSandboxdClientFactory returns the production SandboxdClientFactory: an HTTP +// sandboxd client per node advertise address, over the shared client. +func NewSandboxdClientFactory() SandboxdClientFactory { + hc := NewSandboxdHTTPClient() return func(addr, token string) SandboxdClient { - base := addr - if !strings.Contains(base, "://") { - base = "http://" + base - } - return sandboxd.New(base, token, sandboxd.WithHTTPClient(hc)) + return sandboxd.New(SandboxdBaseURL(addr), token, sandboxd.WithHTTPClient(hc)) } } @@ -175,6 +198,7 @@ type scatterGatherStore struct { log logr.Logger concurrency int watchPoll time.Duration + index *nodeIndex // sandboxdToken is the uniform fleet api_token; sandboxdFactory builds a // per-node sandboxd client. Both are nil/empty until WithClaimRouting is set, @@ -189,6 +213,7 @@ func NewScatterGatherStore(src InventorySource, opts ...StoreOption) *scatterGat src: src, concurrency: 16, watchPoll: time.Second, + index: newNodeIndex(nodeIndexMaxEntries), } for _, o := range opts { o(s) @@ -242,7 +267,7 @@ func (s *scatterGatherStore) List(ctx context.Context, opts ListOptions) (*sandb // is the authoritative view available, so Get returns from it directly rather // than from an eventually-consistent cluster-wide summary. func (s *scatterGatherStore) Get(ctx context.Context, namespace, name string) (*sandboxv1beta1.Sandbox, error) { - found, err := s.findEntry(ctx, "get", func(inv *NodeInventory, i int) bool { + found, err := s.findEntry(ctx, "get", nameKey(namespace, name), func(inv *NodeInventory, i int) bool { ens, ename := splitNamespacedName(inv.Entries[i].Name) return ens == namespace && ename == name }) @@ -257,9 +282,11 @@ func (s *scatterGatherStore) Get(ctx context.Context, namespace, name string) (* // GetByClaimID resolves the sandbox whose node-local claim id satisfies match, // fanning out per node and canceling on the first hit; only the matching entry -// is materialized. An empty namespace matches every namespace. -func (s *scatterGatherStore) GetByClaimID(ctx context.Context, namespace string, match func(claimID string) bool) (*sandboxv1beta1.Sandbox, error) { - found, err := s.findEntry(ctx, "claim-id get", func(inv *NodeInventory, i int) bool { +// is materialized. An empty namespace matches every namespace. id is the +// caller's spelling of the claim id and keys the owning-node index; match owns +// which node-local id it accepts. +func (s *scatterGatherStore) GetByClaimID(ctx context.Context, namespace, id string, match func(claimID string) bool) (*sandboxv1beta1.Sandbox, error) { + found, err := s.findEntry(ctx, "claim-id get", claimKey(namespace, id), func(inv *NodeInventory, i int) bool { if inv.Entries[i].ID == "" || !match(inv.Entries[i].ID) { return false } @@ -305,6 +332,7 @@ func (s *scatterGatherStore) Claim(ctx context.Context, namespace, name string, ClaimRef: namespace + "/" + name, }) if claimErr == nil { + s.index.remember(nameKey(namespace, name), best.node) return Assignment{SandboxName: res.ID, Node: best.node, Address: res.OwnerAddr, Token: res.Token, Deadline: res.Deadline}, nil } if !errors.Is(claimErr, sandboxd.ErrNodeAtCapacity) { @@ -349,10 +377,17 @@ func (s *scatterGatherStore) Watch(ctx context.Context, opts ListOptions) (watch return w, nil } -// findEntry sweeps every node inventory with List's bounded fan-out and returns -// the first entry matching match synthesized as a Sandbox, canceling the rest -// of the sweep on the hit. Nil with a nil error means no entry matched. -func (s *scatterGatherStore) findEntry(ctx context.Context, op string, match func(inv *NodeInventory, i int) bool) (*sandboxv1beta1.Sandbox, error) { +// findEntry resolves the first entry matching match, synthesized as a Sandbox. +// It reads the node the index last saw holding key and only sweeps the whole +// fleet on a miss, canceling the rest of the sweep on the hit. Nil with a nil +// error means no entry matched. +func (s *scatterGatherStore) findEntry(ctx context.Context, op, key string, match func(inv *NodeInventory, i int) bool) (*sandboxv1beta1.Sandbox, error) { + if node, ok := s.index.lookup(key); ok { + if sb := s.matchOnNode(ctx, node, match); sb != nil { + return sb, nil + } + s.index.forget(key) + } nodes, err := s.src.ListNodes(ctx) if err != nil { return nil, fmt.Errorf("scale: enumerate node inventories: %w", err) @@ -374,8 +409,12 @@ func (s *scatterGatherStore) findEntry(ctx context.Context, op string, match fun } inv, err := s.src.NodeInventory(gctx, node) if err != nil { - s.log.V(1).Info("node inventory unavailable during "+op+"; skipping node", - "node", node, "err", err.Error()) + // A sibling goroutine's hit cancels gctx, which fails every read + // still in flight; those are not unavailable nodes. + if gctx.Err() == nil { + s.log.V(1).Info("node inventory unavailable during "+op+"; skipping node", + "node", node, "err", err.Error()) + } return nil } for i := range inv.Entries { @@ -385,6 +424,7 @@ func (s *scatterGatherStore) findEntry(ctx context.Context, op string, match fun mu.Lock() if found == nil { found = entryToSandbox(inv.Node, inv.Entries[i]) + s.index.remember(key, inv.Node) } mu.Unlock() cancel() @@ -397,24 +437,39 @@ func (s *scatterGatherStore) findEntry(ctx context.Context, op string, match fun return found, nil } +// matchOnNode resolves match against one node's inventory, returning nil when +// that node is unreadable or no longer holds the entry. +func (s *scatterGatherStore) matchOnNode(ctx context.Context, node string, match func(inv *NodeInventory, i int) bool) *sandboxv1beta1.Sandbox { + inv, err := s.src.NodeInventory(ctx, node) + if err != nil { + return nil + } + for i := range inv.Entries { + if match(inv, i) { + return entryToSandbox(inv.Node, inv.Entries[i]) + } + } + return nil +} + // warmCandidates lists every node advertising warm capacity for pool, fanning // out per node like List. A node whose inventory is unavailable is skipped, not // fatal: the fleet stays claimable while one node is partitioned. func (s *scatterGatherStore) warmCandidates(ctx context.Context, pool PoolKey) ([]warmCandidate, error) { return fanOutNodes(ctx, s, func(gctx context.Context, n string) []warmCandidate { - inv, err := s.src.NodeInventory(gctx, n) + addr, pools, err := s.src.NodeCapacity(gctx, n) if err != nil { s.log.V(1).Info("node inventory unavailable during claim node-pick; skipping", "node", n, "err", err.Error()) return nil } - if inv.Address == "" { + if addr == "" { return nil } var out []warmCandidate - for j := range inv.Pools { - if pc := inv.Pools[j]; pc.Warm > 0 && poolCapacityMatches(pc, pool) { - out = append(out, warmCandidate{node: n, addr: inv.Address, warm: pc.Warm}) + for j := range pools { + if pc := pools[j]; pc.Warm > 0 && poolCapacityMatches(pc, pool) { + out = append(out, warmCandidate{node: n, addr: addr, warm: pc.Warm}) } } return out @@ -443,8 +498,9 @@ func (s *scatterGatherStore) runWatch(ctx context.Context, opts ListOptions, w * // A fixed cadence, deliberately: backing off while quiet would let a sandbox // that is created and deleted inside the widened gap produce neither an Added - // nor a Deleted. Re-deriving the fleet view costs 1.6ms at 20 nodes (40ms at - // the 200-node projection), which is not worth losing events over. + // nor a Deleted. Re-deriving the fleet view costs 6.5ms at 26 nodes and 2600 + // sandboxes, and 1.2s at the 200x2000 projection, so one watcher per fleet is + // the supported shape. ticker := time.NewTicker(s.watchPoll) defer ticker.Stop() for { @@ -589,42 +645,6 @@ type InventoryApplier interface { Apply(ctx context.Context, inv *NodeInventory) error } -// NodeInventoryPublisher server-side-applies one NodeInventory object for its -// node on a slow cadence, summarizing the node's live sandboxes. This is the -// entire L3 write path: O(nodes) applies, no per-sandbox etcd object. -type NodeInventoryPublisher struct { - node string - live NodeLiveSource - applier InventoryApplier - log logr.Logger -} - -// NewNodeInventoryPublisher builds a publisher for node, reading live state from -// live and applying via applier. -func NewNodeInventoryPublisher(node string, live NodeLiveSource, applier InventoryApplier, log logr.Logger) *NodeInventoryPublisher { - return &NodeInventoryPublisher{node: node, live: live, applier: applier, log: log} -} - -// Publish reads the node's live sandboxes and server-side-applies a single -// NodeInventory object for the node, returning the number of summarized entries. -func (p *NodeInventoryPublisher) Publish(ctx context.Context) (int, error) { - entries, err := p.live.LiveSandboxes(ctx) - if err != nil { - return 0, fmt.Errorf("scale: read node %q live sandboxes: %w", p.node, err) - } - inv := &NodeInventory{ - Kind: NodeInventoryGVK.Kind, - APIVersion: NodeInventoryGVK.GroupVersion().String(), - Name: p.node, - Node: p.node, - Entries: entries, - } - if err := p.applier.Apply(ctx, inv); err != nil { - return 0, fmt.Errorf("scale: apply node %q inventory: %w", p.node, err) - } - return len(entries), nil -} - var _ InventoryApplier = (*ssaInventoryApplier)(nil) type ssaInventoryApplier struct { @@ -734,6 +754,20 @@ func (s *StaticInventorySource) NodeInventory(_ context.Context, node string) (* return inv.DeepCopy(), nil } +// NodeCapacity returns one node's address and pools without copying its entries. +func (s *StaticInventorySource) NodeCapacity(_ context.Context, node string) (string, []PoolCapacity, error) { + s.mu.RLock() + defer s.mu.RUnlock() + if _, ok := s.partition[node]; ok { + return "", nil, fmt.Errorf("scale: node %q partitioned from aggregated server", node) + } + inv, ok := s.inv[node] + if !ok { + return "", nil, fmt.Errorf("scale: no inventory published for node %q", node) + } + return inv.Address, slices.Clone(inv.Pools), nil +} + // ObjectCount is the number of durable NodeInventory objects held — the O(nodes) // etcd object count backing every synthesized sandbox. func (s *StaticInventorySource) ObjectCount() int { @@ -795,6 +829,37 @@ func (s *ClientInventorySource) NodeInventory(ctx context.Context, node string) return inv, nil } +// NodeCapacity decodes only the address and pools fields, so the claim and +// routing paths never pay for the node's whole entry list. +func (s *ClientInventorySource) NodeCapacity(ctx context.Context, node string) (string, []PoolCapacity, error) { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(NodeInventoryGVK) + if err := s.reader.Get(ctx, types.NamespacedName{Name: node}, u); err != nil { + return "", nil, fmt.Errorf("scale: get node %q inventory: %w", node, err) + } + addr, _, err := unstructured.NestedString(u.Object, "address") + if err != nil { + return "", nil, fmt.Errorf("scale: decode node %q address: %w", node, err) + } + raw, _, err := unstructured.NestedSlice(u.Object, "pools") + if err != nil { + return "", nil, fmt.Errorf("scale: decode node %q pools: %w", node, err) + } + pools := make([]PoolCapacity, 0, len(raw)) + for _, item := range raw { + m, ok := item.(map[string]any) + if !ok { + continue + } + pc := PoolCapacity{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(m, &pc); err != nil { + return "", nil, fmt.Errorf("scale: decode node %q pool capacity: %w", node, err) + } + pools = append(pools, pc) + } + return addr, pools, nil +} + // fanOutNodes enumerates the node inventories and runs work per node with // List's bounded concurrency, concatenating per-node results in node order. // A node the work skips contributes nil. @@ -871,6 +936,9 @@ func synthLabels(node string, e InventoryEntry) map[string]string { if e.Phase != "" { l[PhaseLabel] = e.Phase } + if e.Template != "" { + l[TemplateLabel] = e.Template + } if e.ClaimRef != "" { _, claim := splitNamespacedName(e.ClaimRef) if claim != "" { @@ -931,7 +999,7 @@ func readyReason(phase string) string { // unchanged one. It is opaque, as the API contract requires. func resourceVersionFor(ns, name string, e InventoryEntry) string { h := fnv.New64a() - _, _ = h.Write([]byte(ns + "/" + name + "|" + e.ID + "|" + e.Phase + "|" + e.ClaimRef + "|" + e.Address + "|" + deadlineValue(e))) + _, _ = h.Write([]byte(ns + "/" + name + "|" + e.ID + "|" + e.Phase + "|" + e.ClaimRef + "|" + e.Address + "|" + e.Template + "|" + deadlineValue(e))) return strconv.FormatUint(h.Sum64(), 10) } diff --git a/pkg/scale/sandboxstore_impl_test.go b/pkg/scale/sandboxstore_impl_test.go index 32303a6..05d7559 100644 --- a/pkg/scale/sandboxstore_impl_test.go +++ b/pkg/scale/sandboxstore_impl_test.go @@ -29,12 +29,10 @@ func TestScatterGatherList_FlattensAllNodes(t *testing.T) { require.NoError(t, err) require.Len(t, list.Items, 3) - // Sorted by namespace then name. assert.Equal(t, "ns-a/s1", list.Items[0].Namespace+"/"+list.Items[0].Name) assert.Equal(t, "ns-a/s3", list.Items[1].Namespace+"/"+list.Items[1].Name) assert.Equal(t, "ns-b/s2", list.Items[2].Namespace+"/"+list.Items[2].Name) - // The owning node is stamped into status and the node label. byName := map[string]sandboxStatusView{} for _, it := range list.Items { byName[it.Name] = sandboxStatusView{node: it.Status.NodeName, label: it.Labels[NodeLabel]} @@ -96,11 +94,9 @@ func TestScatterGatherList_ToleratesPartitionedNode(t *testing.T) { src := NewStaticInventorySource() src.Put(inv("n1", entry("ns/s1", "Running"))) src.Put(inv("n2", entry("ns/s2", "Running"))) - src.Partition("n2") // listed by ListNodes but its inventory fetch fails + src.Partition("n2") store := NewScatterGatherStore(src, WithLogger(logr.Discard())) - // A partitioned node degrades to eventual consistency: its sandboxes are - // omitted, but the list still succeeds with the reachable node's sandboxes. list, err := store.List(t.Context(), ListOptions{}) require.NoError(t, err) require.Len(t, list.Items, 1) @@ -133,8 +129,7 @@ func TestScatterGatherGet_SynthesizesStatus(t *testing.T) { require.NoError(t, err) assert.Equal(t, []string{"10.1.2.3"}, got.Status.PodIPs) assert.Equal(t, "claim-1", got.Labels[ClaimLabel]) - // The sandboxd claim id rides as an annotation so the apiserver's Delete can - // release exactly this microVM (never by k8s name). + assert.Equal(t, "sb_abc123", got.Annotations[ClaimIDAnnotation]) require.Len(t, got.Status.Conditions, 1) assert.Equal(t, metav1.ConditionTrue, got.Status.Conditions[0].Status) @@ -145,8 +140,6 @@ func TestEntryToSandbox_StampsClaimIDAnnotation(t *testing.T) { withID := entryToSandbox("n1", InventoryEntry{Name: "ns/s1", ID: "sb_abc", Phase: "Running"}) assert.Equal(t, "sb_abc", withID.Annotations[ClaimIDAnnotation]) - // A node that has not published the id yet stamps no annotation, so Delete - // refuses to release rather than guessing by name. noID := entryToSandbox("n1", InventoryEntry{Name: "ns/s1", Phase: "Running"}) _, ok := noID.Annotations[ClaimIDAnnotation] assert.False(t, ok, "expected no claim-id annotation when the entry has no id") @@ -176,11 +169,9 @@ func TestScatterGatherWatch_EmitsAddModifyDelete(t *testing.T) { added := waitForType(t, w, watch.Added, time.Second) assert.Equal(t, "s1", added.Object.(*sandboxv1beta1.Sandbox).Name) - // Phase change → Modified (a content-sensitive ResourceVersion changed). src.Put(inv("n1", entry("ns/s1", "Running"))) waitForType(t, w, watch.Modified, 2*time.Second) - // Entry disappears → Deleted. src.Put(inv("n1")) waitForType(t, w, watch.Deleted, 2*time.Second) } @@ -196,8 +187,7 @@ func TestScatterGather_ObjectCountIsPoolsPlusNodes(t *testing.T) { for i := range perNode { entries = append(entries, entry(fmt.Sprintf("ns/s-%d-%d", k, i), "Running")) } - pub := NewNodeInventoryPublisher(node, entries, src, logr.Discard()) - n, err := pub.Publish(ctx) + n, err := publish(ctx, node, entries, src) require.NoError(t, err) require.Equal(t, perNode, n) } @@ -206,14 +196,11 @@ func TestScatterGather_ObjectCountIsPoolsPlusNodes(t *testing.T) { list, err := store.List(ctx, ListOptions{}) require.NoError(t, err) - // The store serves every sandbox... require.Len(t, list.Items, nodes*perNode) - // ...while the durable object count is O(nodes) NodeInventory (pool intent - // objects are separate and counted in the l3 aggregation evidence): the write - // path is one server-side-apply per node, not per sandbox. + assert.Equal(t, nodes, src.ObjectCount()) assert.Equal(t, nodes, src.ApplyCount()) - // The etcd object count must NOT scale with the sandbox count. + assert.Less(t, src.ObjectCount(), len(list.Items)) } @@ -221,19 +208,15 @@ func TestPublisher_RebuildsFromLiveAfterLoss(t *testing.T) { ctx := t.Context() live := &mutableLive{entries: []InventoryEntry{entry("ns/a", "Running")}} src := NewStaticInventorySource() - pub := NewNodeInventoryPublisher("n1", live, src, logr.Discard()) - - _, err := pub.Publish(ctx) + _, err := publish(ctx, "n1", live, src) require.NoError(t, err) require.Equal(t, 1, src.ObjectCount()) - // The node's NodeInventory object is lost. src.Remove("n1") require.Equal(t, 0, src.ObjectCount()) - // Live state moved on; the next publish rebuilds the object from live state. live.entries = []InventoryEntry{entry("ns/a", "Running"), entry("ns/b", "Running")} - n, err := pub.Publish(ctx) + n, err := publish(ctx, "n1", live, src) require.NoError(t, err) require.Equal(t, 2, n) @@ -250,7 +233,6 @@ func TestNodeInventory_DeepCopyIsIndependent(t *testing.T) { assert.Equal(t, "Running", orig.Entries[0].Phase, "deep copy must not alias entries") assert.Equal(t, "n1", orig.Node) - // DeepCopyObject returns an independent runtime.Object of the same type. obj := orig.DeepCopyObject() require.NotNil(t, obj) clone, ok := obj.(*NodeInventory) @@ -268,8 +250,6 @@ func TestWatchSeesAShortLivedSandbox(t *testing.T) { defer w.Stop() require.Equal(t, watch.Added, (<-w.ResultChan()).Type) - // A sandbox that comes and goes must still be observed. A watch that widened - // its interval while quiet would step over this entirely. src.Put(inv("n1", entry("sb-1", "Running"), entry("sb-2", "Running"))) deadline := time.After(3 * time.Second) @@ -288,19 +268,15 @@ func TestWatchSeesAShortLivedSandbox(t *testing.T) { func TestSSAApplier_UpsertsOneObjectPerNode(t *testing.T) { ctx := t.Context() cli := fake.NewClientBuilder().WithScheme(newScaleScheme(t)).Build() - pub := NewNodeInventoryPublisher("n1", sliceLive{entry("ns/a", "Running")}, - NewSSAInventoryApplier(cli, "vk-test"), logr.Discard()) - - _, err := pub.Publish(ctx) + _, err := publish(ctx, "n1", sliceLive{entry("ns/a", "Running")}, NewSSAInventoryApplier(cli, "vk-test")) require.NoError(t, err) got := &extv1beta1.NodeInventory{} require.NoError(t, cli.Get(ctx, client.ObjectKey{Name: "n1"}, got)) require.Len(t, got.Entries, 1) - pub = NewNodeInventoryPublisher("n1", sliceLive{entry("ns/a", "Running"), entry("ns/b", "Running")}, - NewSSAInventoryApplier(cli, "vk-test"), logr.Discard()) - _, err = pub.Publish(ctx) + _, err = publish(ctx, "n1", sliceLive{entry("ns/a", "Running"), entry("ns/b", "Running")}, + NewSSAInventoryApplier(cli, "vk-test")) require.NoError(t, err) list := &extv1beta1.NodeInventoryList{} @@ -309,6 +285,20 @@ func TestSSAApplier_UpsertsOneObjectPerNode(t *testing.T) { assert.Len(t, list.Items[0].Entries, 2) } +func publish(ctx context.Context, node string, live NodeLiveSource, applier InventoryApplier) (int, error) { + entries, err := live.LiveSandboxes(ctx) + if err != nil { + return 0, err + } + return len(entries), applier.Apply(ctx, &NodeInventory{ + Kind: NodeInventoryGVK.Kind, + APIVersion: NodeInventoryGVK.GroupVersion().String(), + Name: node, + Node: node, + Entries: entries, + }) +} + func inv(node string, entries ...InventoryEntry) *NodeInventory { return &NodeInventory{ Name: node, @@ -319,14 +309,12 @@ func inv(node string, entries ...InventoryEntry) *NodeInventory { func entry(name, phase string) InventoryEntry { return InventoryEntry{Name: name, Phase: phase} } -// sliceLive is a NodeLiveSource fed from an in-memory slice. type sliceLive []InventoryEntry func (s sliceLive) LiveSandboxes(context.Context) ([]InventoryEntry, error) { return []InventoryEntry(s), nil } -// mutableLive is a NodeLiveSource whose entries can change between publishes. type mutableLive struct{ entries []InventoryEntry } func (m *mutableLive) LiveSandboxes(context.Context) ([]InventoryEntry, error) { @@ -335,7 +323,6 @@ func (m *mutableLive) LiveSandboxes(context.Context) ([]InventoryEntry, error) { type sandboxStatusView struct{ node, label string } -// waitForType drains events until one of type want arrives or the deadline hits. func waitForType(t *testing.T, w watch.Interface, want watch.EventType, timeout time.Duration) watch.Event { t.Helper() deadline := time.After(timeout) diff --git a/pkg/scale/sandboxstore_lifecycle.go b/pkg/scale/sandboxstore_lifecycle.go index 5ae83f5..d545ed0 100644 --- a/pkg/scale/sandboxstore_lifecycle.go +++ b/pkg/scale/sandboxstore_lifecycle.go @@ -100,41 +100,6 @@ func (s *scatterGatherStore) DeleteSnapshot(ctx context.Context, node, snapshotI return nil } -// ClaimSnapshot delivers a fresh sandbox branched from a checkpoint. -func (s *scatterGatherStore) ClaimSnapshot(ctx context.Context, node, snapshotID string, ttlSeconds int) (Assignment, error) { - cl, err := s.nodeClient(ctx, node, "claim snapshot", snapshotID) - if err != nil { - return Assignment{}, err - } - res, err := cl.ClaimCheckpoint(ctx, snapshotID, sandboxd.CheckpointClaimSpec{TTLSeconds: ttlSeconds}) - if err != nil { - return Assignment{}, fmt.Errorf("scale: sandboxd claim snapshot %q on node %q: %w", snapshotID, node, err) - } - return Assignment{ - SandboxName: res.ID, - Node: node, - Address: res.OwnerAddr, - Token: res.Token, - Deadline: res.Deadline, - }, nil -} - -// Promote publishes a sandbox as a node-local template. -func (s *scatterGatherStore) Promote(ctx context.Context, node, id, template string) (PoolKey, error) { - if template == "" { - return PoolKey{}, fmt.Errorf("scale: promote requires a template name") - } - cl, err := s.nodeClient(ctx, node, "promote", id) - if err != nil { - return PoolKey{}, err - } - key, err := cl.Promote(ctx, id, sandboxd.PromoteSpec{Template: template}) - if err != nil { - return PoolKey{}, fmt.Errorf("scale: sandboxd promote of %q on node %q: %w", id, node, err) - } - return PoolKey{Template: key.Template, Net: key.Net, Size: key.Size}, nil -} - // Stats reports a sandbox's resource usage from its owning node. func (s *scatterGatherStore) Stats(ctx context.Context, node, id string) (SandboxStats, error) { cl, err := s.nodeClient(ctx, node, "stats", id) @@ -165,14 +130,14 @@ func (s *scatterGatherStore) nodeClient(ctx context.Context, node, verb, id stri if node == "" { return nil, fmt.Errorf("scale: %s requires an owning node", verb) } - inv, err := s.src.NodeInventory(ctx, node) + addr, _, err := s.src.NodeCapacity(ctx, node) if err != nil { return nil, fmt.Errorf("scale: resolve node %q for %s of %q: %w", node, verb, id, err) } - if inv.Address == "" { + if addr == "" { return nil, fmt.Errorf("scale: node %q advertises no sandboxd address for %s of %q", node, verb, id) } - return s.sandboxdFactory(inv.Address, s.sandboxdToken), nil + return s.sandboxdFactory(addr, s.sandboxdToken), nil } // snapshotFrom converts a node's checkpoint record to the store's shape. diff --git a/pkg/scale/warmpool/driver.go b/pkg/scale/warmpool/driver.go index 12a47ed..eb1a9b3 100644 --- a/pkg/scale/warmpool/driver.go +++ b/pkg/scale/warmpool/driver.go @@ -36,10 +36,6 @@ import ( ) const ( - // netAnnotation selects the pool network mode; it mirrors the aggregated - // apiserver's NetAnnotation so a Create derives the same key the driver sets. - netAnnotation = "sandbox.cocoonstack.io/net" - // defaultInterval is the pool resync cadence, and with it the sampling period of // the fleet-wide warm count reported in pool status. Pools are O(single-digit) // and every tick's node fan-out is the same PUT the driver already owes, so a 5s @@ -66,9 +62,14 @@ type PoolSetter interface { // uniform fleet api_token. type ClientFactory func(addr, token string) PoolSetter -// NewSandboxdFactory returns the production factory backed by the real HTTP client. +// NewSandboxdFactory returns the production factory. It shares the store's +// address rendering and keep-alive client, so a node advertising a scheme is +// reachable here too. func NewSandboxdFactory() ClientFactory { - return func(addr, token string) PoolSetter { return sandboxd.New("http://"+addr, token) } + hc := scale.NewSandboxdHTTPClient() + return func(addr, token string) PoolSetter { + return sandboxd.New(scale.SandboxdBaseURL(addr), token, sandboxd.WithHTTPClient(hc)) + } } // Options configures a Driver. @@ -214,19 +215,19 @@ func (d *Driver) schedulableNodes(ctx context.Context) ([]nodeView, error) { } views := make([]nodeView, 0, len(names)) for _, name := range names { - inv, err := d.inv.NodeInventory(ctx, name) + addr, pools, err := d.inv.NodeCapacity(ctx, name) if err != nil { d.log.V(1).Info("skip node without readable inventory", "node", name, "err", err.Error()) continue } - if inv.Address == "" { + if addr == "" { continue } - warmBy := make(map[scale.PoolKey]int, len(inv.Pools)) - for _, pc := range inv.Pools { + warmBy := make(map[scale.PoolKey]int, len(pools)) + for _, pc := range pools { warmBy[scale.PoolKey{Template: pc.Template, Net: pc.Net, Size: pc.Size}] = pc.Warm } - views = append(views, nodeView{name: name, addr: inv.Address, warmBy: warmBy}) + views = append(views, nodeView{name: name, addr: addr, warmBy: warmBy}) } slices.SortFunc(views, func(a, b nodeView) int { return cmp.Compare(a.name, b.name) }) return views, nil @@ -243,10 +244,7 @@ func (d *Driver) poolKey(ctx context.Context, p *extv1beta1.SandboxWarmPool) (sc if err := d.kube.Get(ctx, types.NamespacedName{Namespace: p.Namespace, Name: name}, &tmpl); err != nil { return scale.PoolKey{}, fmt.Errorf("get SandboxTemplate %s/%s: %w", p.Namespace, name, err) } - net := tmpl.Annotations[netAnnotation] - if net == "" { - net = tmpl.Spec.PodTemplate.ObjectMeta.Annotations[netAnnotation] - } + net := scale.NetForAnnotations(tmpl.Annotations, tmpl.Spec.PodTemplate.ObjectMeta.Annotations) return scale.PoolKeyFor(tmpl.Spec.PodTemplate.Spec.Containers, net), nil } diff --git a/pkg/scale/warmpool/driver_test.go b/pkg/scale/warmpool/driver_test.go index 135eaea..b1884bb 100644 --- a/pkg/scale/warmpool/driver_test.go +++ b/pkg/scale/warmpool/driver_test.go @@ -20,11 +20,6 @@ import ( const testImage = "ghcr.io/cocoonstack/sandbox/rt@sha256:deadbeef" -// TestReconcileDistributesAndMatchesPoolKey pins the two load-bearing invariants: -// (1) a SandboxWarmPool's replicas are spread across all nodes summing to EXACTLY -// replicas, and (2) the pool key the driver sets is byte-identical to what the -// aggregated apiserver derives for a Sandbox with the same image — so a Create -// always finds the warm capacity (the G5 fix). A drift here means perpetual 503s. func TestReconcileDistributesAndMatchesPoolKey(t *testing.T) { d, setter, inv, kube := newTestDriver(t, warmPool("p", 100), template()) putNodes(inv, 26) @@ -33,7 +28,6 @@ func TestReconcileDistributesAndMatchesPoolKey(t *testing.T) { t.Fatalf("reconcile: %v", err) } - // Exactly 26 nodes were PUT, summing to exactly 100, spread evenly. if len(setter.byAddr) != 26 { t.Fatalf("PUT %d nodes, want 26", len(setter.byAddr)) } @@ -63,8 +57,6 @@ func TestReconcileDistributesAndMatchesPoolKey(t *testing.T) { t.Fatalf("uneven spread: min=%d max=%d", minWarm, maxWarm) } - // Status is written back from the warm each node reported (0 here — targets - // accepted, nothing refilled yet). var got extv1beta1.SandboxWarmPool if err := kube.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "p"}, &got); err != nil { t.Fatalf("get pool: %v", err) @@ -74,12 +66,10 @@ func TestReconcileDistributesAndMatchesPoolKey(t *testing.T) { } } -// TestReconcileWritesWarmStatus verifies status reflects the live warm each node -// reports in its PUT response. func TestReconcileWritesWarmStatus(t *testing.T) { d, setter, inv, kube := newTestDriver(t, warmPool("p", 8), template()) key := scale.PoolKeyFor([]corev1.Container{{Image: testImage}}, "") - // Two nodes each already reporting 4 warm of the pool's key → status 8. + for _, name := range []string{"a", "b"} { inv.Put(&scale.NodeInventory{ Name: name, @@ -101,24 +91,19 @@ func TestReconcileWritesWarmStatus(t *testing.T) { } } -// TestStatusPrefersPutResponseOverStaleInventory pins WHY status is sourced from -// the PUT response: NodeInventory is published on its own cadence (30s by -// default) and is read before this tick's apply, so a fill in progress reads -// low. Sampling the response instead makes the reported total as fresh as the -// resync interval — that equality is what the 5s sampling period rests on. func TestStatusPrefersPutResponseOverStaleInventory(t *testing.T) { d, setter, inv, kube := newTestDriver(t, warmPool("p", 20), template()) key := scale.PoolKeyFor([]corev1.Container{{Image: testImage}}, "") for _, name := range []string{"a", "b"} { addr := "10.0.0." + name + ":7777" - // Inventory still carries the pre-fill snapshot: 4 warm per node. + inv.Put(&scale.NodeInventory{ Name: name, Node: name, Address: addr, Pools: []extv1beta1.PoolCapacity{{Template: key.Template, Net: key.Net, Size: key.Size, Warm: 4, Target: 10}}, }) - // The node itself now reports 7 — the truth as of this tick. + setter.reportWarm(addr, 7) } if err := d.reconcileOnce(t.Context()); err != nil { @@ -133,9 +118,6 @@ func TestStatusPrefersPutResponseOverStaleInventory(t *testing.T) { } } -// TestStatusFallsBackToInventoryWhenPutFails pins that a node the driver could -// not reach keeps contributing its last known inventory warm, so one unreachable -// node cannot make the fleet total collapse toward zero. func TestStatusFallsBackToInventoryWhenPutFails(t *testing.T) { d, setter, inv, kube := newTestDriver(t, warmPool("p", 20), template()) key := scale.PoolKeyFor([]corev1.Container{{Image: testImage}}, "") @@ -162,11 +144,7 @@ func TestStatusFallsBackToInventoryWhenPutFails(t *testing.T) { } } -// TestTwoPoolsSameKeyAggregate pins that two SandboxWarmPools resolving to the -// same pool key produce ONE spec per node with SUMMED warm — sandboxd rejects a -// PUT that repeats a key ("duplicate pool"), which silently stalled every node. func TestTwoPoolsSameKeyAggregate(t *testing.T) { - // Both pools reference the same template "tpl" → identical key. d, setter, inv, _ := newTestDriver(t, warmPool("p1", 3), warmPool("p2", 5), template()) putNodes(inv, 2) if err := d.reconcileOnce(t.Context()); err != nil { @@ -174,8 +152,7 @@ func TestTwoPoolsSameKeyAggregate(t *testing.T) { } sum := 0 for addr, specs := range setter.byAddr { - // The load-bearing invariant: ONE spec per node (keys aggregated, no - // duplicate the PUT would reject), and every spec carries the shared key. + if len(specs) != 1 { t.Fatalf("node %s got %d specs, want 1 aggregated (no duplicate key): %+v", addr, len(specs), specs) } @@ -184,14 +161,12 @@ func TestTwoPoolsSameKeyAggregate(t *testing.T) { } sum += specs[0].Warm } - // p1(3) + p2(5) summed across the fleet = 8, regardless of per-node rounding. + if sum != 8 { t.Fatalf("fleet warm total = %d, want 8 (3+5 summed)", sum) } } -// TestDrainOnZeroReplicas: replicas=0 sets every node's target to 0 (the -// control-plane drain — kubectl scale to 0 or delete recycles the pool). func TestDrainOnZeroReplicas(t *testing.T) { d, setter, inv, _ := newTestDriver(t, warmPool("p", 0), template()) putNodes(inv, 3) @@ -216,8 +191,6 @@ func TestApplyBoundsEachNodeCall(t *testing.T) { } } -// BenchmarkReconcileOnce measures one global driver pass — the cost of every -// wake-up, spurious or not — at the deployed fleet shape (26 nodes, 4 pools). func BenchmarkReconcileOnce(b *testing.B) { objs := []client.Object{template()} for i := range 4 { @@ -234,20 +207,16 @@ func BenchmarkReconcileOnce(b *testing.B) { } } -// fakeSetter records the last pools set per node address and plays back the warm -// counts a node reports in its PUT response — the driver's status source. type fakeSetter struct { mu sync.Mutex byAddr map[string][]sandboxd.PoolSpec - // warm is the warm count each node echoes per pool; absent means 0 (target - // accepted, nothing filled yet). + warm map[string]int - // failAddr, when set, makes that node's PUT fail. + failAddr string sawDeadline bool } -// reportWarm makes addr answer every PUT reporting n warm for each pool it holds. func (f *fakeSetter) reportWarm(addr string, n int) { f.mu.Lock() defer f.mu.Unlock() @@ -274,14 +243,13 @@ func (n *fakeNode) SetPools(ctx context.Context, pools []sandboxd.PoolSpec) (*sa return nil, errors.New("node unreachable") } n.parent.byAddr[n.addr] = pools - // Mirror real sandboxd: the response echoes every pool the node now holds, - // each with its live warm count. + info := &sandboxd.NodeInfo{} for _, p := range pools { info.Pools = append(info.Pools, sandboxd.NodePool{ Key: sandboxd.PoolKey{Template: p.Template, Net: p.Net, Size: p.Size}, Warm: n.parent.warm[n.addr], - // Target echoes the requested warm watermark. + Target: p.Warm, }) } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index a4b14e0..bba6dca 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -14,6 +14,7 @@ import ( "flag" "fmt" "os" + "slices" "strings" "time" @@ -208,10 +209,10 @@ func isStandardNode(ctx context.Context, nodeName string) (bool, error) { return false, nil } // vk-cocoon nodes carry the provider taint. - for _, t := range n.Spec.Taints { - if t.Key == "virtual-kubelet.io/provider" { - return false, nil - } + if slices.ContainsFunc(n.Spec.Taints, func(t corev1.Taint) bool { + return t.Key == "virtual-kubelet.io/provider" + }) { + return false, nil } return true, nil } @@ -273,12 +274,9 @@ func scCoreCreateReady(ctx context.Context) (string, error) { if err != nil { return "", fmt.Errorf("backing pod not found: %w", err) } - owned := false - for _, o := range pod.OwnerReferences { - if o.Kind == "Sandbox" && o.Name == name { - owned = true - } - } + owned := slices.ContainsFunc(pod.OwnerReferences, func(o metav1.OwnerReference) bool { + return o.Kind == "Sandbox" && o.Name == name + }) if !owned { return "", fmt.Errorf("pod not owned by Sandbox") } @@ -327,10 +325,10 @@ func scNoVKInjection(ctx context.Context) (string, error) { if v := pod.Spec.NodeSelector["node.kubernetes.io/instance-type"]; v == "virtual-node" { return "", fmt.Errorf("vk nodeSelector injected") } - for _, t := range pod.Spec.Tolerations { - if t.Key == "virtual-kubelet.io/provider" { - return "", fmt.Errorf("vk toleration injected") - } + if slices.ContainsFunc(pod.Spec.Tolerations, func(t corev1.Toleration) bool { + return t.Key == "virtual-kubelet.io/provider" + }) { + return "", fmt.Errorf("vk toleration injected") } for k := range pod.Annotations { if strings.HasPrefix(k, "cocoonset.cocoonstack.io/") || strings.HasPrefix(k, "vm.cocoonstack.io/") { @@ -352,7 +350,6 @@ func scSuspendResume(ctx context.Context) (string, error) { if _, err := waitSandboxReady(ctx, name); err != nil { return "", err } - // Suspend s := &sandboxv1beta1.Sandbox{} if err := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, s); err != nil { return "", err @@ -364,7 +361,6 @@ func scSuspendResume(ctx context.Context) (string, error) { if err := waitPodGone(ctx, name, 90*time.Second); err != nil { return "", fmt.Errorf("pod not removed after suspend: %w", err) } - // Resume if err := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, s); err != nil { return "", err } @@ -499,7 +495,6 @@ func scTemplateCRUD(ctx context.Context) (string, error) { if err := cl.Create(ctx, t); err != nil { return "", err } - // update: add an env injection policy got := &extv1beta1.SandboxTemplate{} if err := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, got); err != nil { return "", err @@ -527,7 +522,6 @@ func scWarmPoolScale(ctx context.Context) (string, error) { if err := cl.Create(ctx, wp); err != nil { return "", err } - // wait for warm sandboxes to appear deadline := time.Now().Add(150 * time.Second) for time.Now().Before(deadline) { sl := &sandboxv1beta1.SandboxList{} @@ -583,7 +577,6 @@ func scClaimWarmHit(ctx context.Context) (string, error) { func scConversion(ctx context.Context) (string, error) { name := "e2e-conv" - // create via v1alpha1 _ = cl.Delete(ctx, &sandboxv1alpha1.Sandbox{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: *ns}}) time.Sleep(2 * time.Second) a := &sandboxv1alpha1.Sandbox{ @@ -606,7 +599,6 @@ func scConversion(ctx context.Context) (string, error) { if len(b.Spec.PodTemplate.Spec.Containers) == 0 || b.Spec.PodTemplate.Spec.Containers[0].Image != image { return "", fmt.Errorf("conversion lost podTemplate") } - // read back as v1alpha1 too a2 := &sandboxv1alpha1.Sandbox{} if err := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, a2); err != nil { return "", fmt.Errorf("read as v1alpha1: %w", err) diff --git a/test/e2ebench/main.go b/test/e2ebench/main.go index b2d5bb3..9851b06 100644 --- a/test/e2ebench/main.go +++ b/test/e2ebench/main.go @@ -94,14 +94,12 @@ func main() { "path": "admission webhook → SandboxWarmPool(replicas=N) → agents.x-k8s.io Sandbox → vk-cocoon microVM → SandboxClaim adopt(Bound) → owner-authorized delete → reclaim; pure Kubernetes, no proprietary control plane", } - // (1) prod-desktop baseline on the node — must be unchanged at the end. prodBefore := podCount(ctx, *prodNS, *node) fmt.Printf("[prod] baseline: %d desktop pods on %s\n", prodBefore, *node) ensureNS(ctx) ensureTemplate(ctx) - // (2) Fill the pool: admission admits N Sandbox CRs → N real microVMs. fmt.Printf("[fill] creating pool %s replicas=%d on %s\n", poolName, *poolSize, *node) ensurePool(ctx, int32(*poolSize)) admissionPass := true @@ -111,7 +109,6 @@ func main() { fmt.Printf("[fill] WARN only %d/%d ready before timeout\n", filled, *poolSize) } - // (3) Four-way cross-check. rr, crc, pods := crossCheck(ctx) cross := map[string]any{ "warmpool_ready_replicas": rr, "sandbox_cr_count": crc, @@ -120,7 +117,6 @@ func main() { res["cross_checks"] = cross fmt.Printf("[cross] readyReplicas=%d sandboxCR=%d pods=%d\n", rr, crc, pods) - // (4) Fire N claims through admission, wait for Bound. bound, createFails := fireClaims(ctx, *poolSize, *claimConc, *claimWait) res["success"] = bound if createFails > 0 { @@ -129,13 +125,11 @@ func main() { res["admission_pass"] = admissionPass && bound == *poolSize && rr == *poolSize && crc == *poolSize fmt.Printf("[claim] bound=%d/%d createFails=%d\n", bound, *poolSize, createFails) - // (5) Release + cleanup: delete claims then pool (owner-authorized), wait 0 leak. releasePass, leaked := releaseAndCleanup(ctx, *cleanupWait) res["release_pass"] = releasePass res["leaked"] = leaked fmt.Printf("[cleanup] releasePass=%v leaked=%d\n", releasePass, leaked) - // (6) prod-desktops intact. prodAfter := podCount(ctx, *prodNS, *node) res["prod_intact"] = prodAfter res["prod_before"] = prodBefore @@ -280,7 +274,6 @@ func fireClaims(ctx context.Context, n, conc, timeoutSec int) (bound, createFail // and template, and waits until every Sandbox CR and pod this run created on the // node reaches 0. Returns whether release completed and the residual leak count. func releaseAndCleanup(ctx context.Context, timeoutSec int) (releasePass bool, leaked int) { - // Delete claims first (release), then the pool (cascade-delete warm sandboxes). cll := &extv1beta1.SandboxClaimList{} if err := cl.List(ctx, cll, ctrlclient.InNamespace(*ns)); err == nil { for i := range cll.Items { diff --git a/test/l2bench/main.go b/test/l2bench/main.go index 83861b6..693a1b4 100644 --- a/test/l2bench/main.go +++ b/test/l2bench/main.go @@ -211,7 +211,6 @@ func main() { fs := newFakeSandboxd() defer fs.srv.Close() - // --- claim-path latency (gateway overhead must be sub-millisecond) --- gw := scale.NewGateway(scale.GatewayConfig{ Node: node, Client: sandboxd.New(fs.srv.URL, "root-token"), Authorizer: allowAuthorizer{}, Recorder: &countingRecorder{}, @@ -221,7 +220,6 @@ func main() { p50 := pct(lat, 50) p95 := pct(lat, 95) - // --- orphan-binding convergence (adopt-only; zero VM destroys) --- reconciled, remaining := injectAndReconcileOrphans(ctx, fs, *orphansFlag) destroys := fs.releases.Load() diff --git a/test/l3bench/main.go b/test/l3bench/main.go index 0bcdc25..71836af 100644 --- a/test/l3bench/main.go +++ b/test/l3bench/main.go @@ -85,7 +85,6 @@ func main() { } wantSandboxes := nodes * perNode - // --- durable etcd stand-in: NodeInventory objects + WarmPool intent --------- // The source is both the publisher's apply target (the O(nodes) write path) // and the store's read source (cache-fed NodeInventory read). No per-sandbox // object is ever created. @@ -120,11 +119,15 @@ func main() { Address: fmt.Sprintf("10.%d.%d.%d:7777", k, (i>>8)&0xff, i&0xff), }) } - pub := scale.NewNodeInventoryPublisher(node, entries, source, logr.Discard()) - n, err := pub.Publish(ctx) - must(err) - if n != perNode { - fail("publisher summarized %d entries for %s, want %d", n, node, perNode) + must(source.Apply(ctx, &scale.NodeInventory{ + Kind: scale.NodeInventoryGVK.Kind, + APIVersion: scale.NodeInventoryGVK.GroupVersion().String(), + Name: node, + Node: node, + Entries: entries, + })) + if n := len(entries); n != perNode { + fail("published %d entries for %s, want %d", n, node, perNode) } } @@ -140,7 +143,6 @@ func main() { fail("etcd object count %d != nodes+pools (%d+%d)", etcdObjectCount, nodes, pools) } - // --- stand up the aggregated apiserver in-process --------------------------- store := scale.NewScatterGatherStore(source, scale.WithLogger(logr.Discard()), scale.WithWatchPollInterval(50*time.Millisecond)) server, err := sandboxapiserver.NewInProcessServer("l3bench-apiserver", store) must(err) @@ -149,7 +151,6 @@ func main() { rc := newRESTClient(ts.URL) - // (1) kubectl-equivalent cluster-scoped list: GET /apis/agents.x-k8s.io/v1beta1/sandboxes allList := &sandboxv1beta1.SandboxList{} if err := rc.Get().Resource("sandboxes").Do(ctx).Into(allList); err != nil { fail("client-go cluster-scoped list failed: %v", err) @@ -158,7 +159,6 @@ func main() { fail("cluster list returned %d sandboxes, want %d", len(allList.Items), wantSandboxes) } - // (2) kubectl-equivalent namespaced list: GET .../namespaces//sandboxes nsList := &sandboxv1beta1.SandboxList{} if err := rc.Get().Namespace(sampleNS).Resource("sandboxes").Do(ctx).Into(nsList); err != nil { fail("client-go namespaced list failed: %v", err) @@ -169,7 +169,6 @@ func main() { fail("namespaced list returned %d, want %d (>0)", len(nsList.Items), len(wantNS.Items)) } - // (3) per-item Get (owning-node routing): GET .../namespaces//sandboxes/ got := &sandboxv1beta1.Sandbox{} if err := rc.Get().Namespace(sampleNS).Resource("sandboxes").Name(sampleName).Do(ctx).Into(got); err != nil { fail("client-go get failed: %v", err) @@ -178,7 +177,6 @@ func main() { fail("get returned %s/%s, want %s/%s", got.Namespace, got.Name, sampleNS, sampleName) } - // (4) label-selected list proves selector fan-out honoring. labelList := &sandboxv1beta1.SandboxList{} if err := rc.Get().Resource("sandboxes").Param("labelSelector", scale.NodeLabel+"=node-0").Do(ctx).Into(labelList); err != nil { fail("client-go label-selected list failed: %v", err) @@ -187,7 +185,6 @@ func main() { fail("label-selected list returned %d, want %d", len(labelList.Items), perNode) } - // (5) watch merge (best-effort): read at least one event off the merged // stream, narrowed to the sample object so the initial sync is a single event. watchEvents, watchOK := exerciseWatch(ctx, rc, sampleNS, sampleName) diff --git a/test/poolbench/main.go b/test/poolbench/main.go index 98b3579..89928e7 100644 --- a/test/poolbench/main.go +++ b/test/poolbench/main.go @@ -96,7 +96,6 @@ func main() { ensureNS(ctx) ensureTemplate(ctx) - // ---- Phase A: pool fill throughput ---- fill := fillPool(ctx, *poolSize) result["fill"] = fill fmt.Printf("[fill] target=%d reachedReady=%d in %.1fs -> %.2f ready/s; createToReady p50=%.0fms p95=%.0fms\n", @@ -112,7 +111,6 @@ func main() { // Gate: the pool must be fully warm and steady, else claims race replenishment. waitPoolStable(ctx, *poolSize, 120) - // ---- Phase B: warm-claim latency ---- claimRes := claimLatency(ctx, *claims, *claimConc) result["claim"] = claimRes fmt.Printf("[claim] n=%d conc=%d warmHits=%d succ=%.1f%% latency p50=%.0fms p95=%.0fms p99=%.0fms max=%.0fms\n", diff --git a/test/scalebench/main.go b/test/scalebench/main.go index 98c45d1..7189833 100644 --- a/test/scalebench/main.go +++ b/test/scalebench/main.go @@ -49,9 +49,9 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" ctrls "github.com/cocoonstack/sandbox-operator/extensions/controllers" - "github.com/cocoonstack/sandbox-operator/extensions/controllers/queue" "github.com/cocoonstack/sandbox-operator/internal/hash" asmetrics "github.com/cocoonstack/sandbox-operator/internal/metrics" + "github.com/cocoonstack/sandbox-operator/internal/queue" "github.com/cocoonstack/sandbox-operator/test/benchutil" ) diff --git a/test/scalestress/main.go b/test/scalestress/main.go index c9812ee..bbf07b2 100644 --- a/test/scalestress/main.go +++ b/test/scalestress/main.go @@ -178,7 +178,7 @@ func main() { result["rounds"] = rounds result["aborted"] = abort - // Interpretation. A wedge tendency shows up as: any NEW time-out rejection on + // A wedge tendency shows up as: any NEW time-out rejection on // the vk LIST level during the ramp, seat use approaching the nominal limit, or // LIST latency climbing materially with object count. newRejections, maxInUse := 0.0, 0.0