diff --git a/cmd/sandbox-operator/main.go b/cmd/sandbox-operator/main.go index c210b44..25fb0c7 100644 --- a/cmd/sandbox-operator/main.go +++ b/cmd/sandbox-operator/main.go @@ -38,6 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -228,6 +229,9 @@ func (o *options) run() error { return fmt.Errorf("start manager: %w", err) } + if err := asmetrics.Register(metrics.Registry); err != nil { + return err + } asmetrics.RegisterSandboxCollector(ctx, mgr.GetClient(), mgr.GetLogger().WithName("sandbox-collector")) if err := o.setupControllers(mgr, instrumenter, podMutator); err != nil { diff --git a/examples/lifecycle/example.go b/examples/lifecycle/example.go index eabcbe5..3f4826b 100644 --- a/examples/lifecycle/example.go +++ b/examples/lifecycle/example.go @@ -50,6 +50,7 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" + "github.com/cocoonstack/sandbox-operator/pkg/scale" ) const ( @@ -58,8 +59,7 @@ const ( // publish does not fail the walk-through. visibilityTimeout = 90 * time.Second - // claimIDAnnotation carries the node-local claim id of a delivered sandbox. - claimIDAnnotation = "sandbox.cocoonstack.io/claim-id" + claimIDAnnotation = scale.ClaimIDAnnotation ) type options struct { @@ -176,13 +176,11 @@ func runKubernetes(ctx context.Context, c client.Client, rc rest.Interface, o op } sb.Spec.PodTemplate.Spec.Containers = []corev1.Container{{Name: "agent", Image: o.template}} - // 1. Create — a claim against a warm pool, not a scheduling decision. if err := c.Create(ctx, sb); err != nil { return fmt.Errorf("create Sandbox: %w", err) } stepf("create", "Sandbox %s/%s", o.namespace, name) - // 2. Wait until the read view publishes it, then Get and List. live, err := waitVisible(ctx, c, o.namespace, name) if err != nil { return err @@ -195,7 +193,6 @@ func runKubernetes(ctx context.Context, c client.Client, rc rest.Interface, o op } stepf("list", "%d sandbox(es) in %s", len(list.Items), o.namespace) - // 3. snapshot — an immutable checkpoint; the source keeps running. snap := &sandboxv1beta1.SandboxSnapshotResult{} if err := post(ctx, rc, o.namespace, name, "snapshot", &sandboxv1beta1.SandboxSnapshotOptions{Name: "example-checkpoint"}, snap); err != nil { @@ -203,8 +200,6 @@ func runKubernetes(ctx context.Context, c client.Client, rc rest.Interface, o op } stepf("snapshot", "snapshotID=%s on node=%s", snap.SnapshotID, snap.NodeName) - // 4. fork — the source is checkpointed in place and keeps running; each - // child is a brand-new sandbox with its own id and lease. forked := &sandboxv1beta1.SandboxForkResult{} if err := post(ctx, rc, o.namespace, name, "fork", &sandboxv1beta1.SandboxForkOptions{Count: 2, TTLSeconds: 600}, forked); err != nil { @@ -214,23 +209,18 @@ func runKubernetes(ctx context.Context, c client.Client, rc rest.Interface, o op stepf("fork", "child[%d] sandboxID=%s node=%s", i, child.SandboxID, child.NodeName) } - // 5. pause — writes the guest's memory out and stops the VM, so this is - // the slow verb: its cost is proportional to guest RAM. start := time.Now() if err := post(ctx, rc, o.namespace, name, "pause", &sandboxv1beta1.SandboxPauseOptions{}, nil); err != nil { return fmt.Errorf("pause: %w", err) } stepf("pause", "took %s (proportional to guest memory)", time.Since(start).Round(time.Millisecond)) - // 6. resume — cocoon's mmap restore fast path, and idempotent on a - // sandbox that is already running. start = time.Now() if err := post(ctx, rc, o.namespace, name, "resume", &sandboxv1beta1.SandboxResumeOptions{}, nil); err != nil { return fmt.Errorf("resume: %w", err) } stepf("resume", "took %s (mmap restore fast path)", time.Since(start).Round(time.Millisecond)) - // 7. Delete — releases the claim back to the node's pool. if o.keep { stepf("delete", "skipped (-keep)") return nil @@ -247,25 +237,21 @@ func runKubernetes(ctx context.Context, c client.Client, rc rest.Interface, o op // served from NodeInventory, which is republished on a ~30s cadence — so a // caller that reads immediately after creating must expect a NotFound. func waitVisible(ctx context.Context, c client.Client, ns, name string) (*sandboxv1beta1.Sandbox, error) { - deadline := time.Now().Add(visibilityTimeout) - for { - var sb sandboxv1beta1.Sandbox + var sb sandboxv1beta1.Sandbox + err := pollVisible(ctx, fmt.Sprintf("sandbox %s/%s", ns, name), func() (bool, error) { err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &sb) - if err == nil { - return &sb, nil - } - if !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("get Sandbox: %w", err) + if apierrors.IsNotFound(err) { + return false, nil } - if time.Now().After(deadline) { - return nil, fmt.Errorf("sandbox %s/%s not visible within %s", ns, name, visibilityTimeout) - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(3 * time.Second): + if err != nil { + return false, fmt.Errorf("get Sandbox: %w", err) } + return true, nil + }) + if err != nil { + return nil, err } + return &sb, nil } // post invokes an action subresource. These are POST-only verbs (the @@ -313,15 +299,12 @@ func runE2B(ctx context.Context, o options) error { } stepf("health", "reachable") - // 1. Templates — the pools this fleet can serve claims from; a templateID - // from here is what create accepts. var templates []map[string]any if err := e.do(ctx, http.MethodGet, "/templates", nil, &templates); err != nil { return fmt.Errorf("list templates: %w", err) } stepf("templates", "%d available", len(templates)) - // 2. Create. var created map[string]any if err := e.do(ctx, http.MethodPost, "/sandboxes", map[string]any{"templateID": o.template, "timeout": 600}, &created); err != nil { @@ -337,7 +320,6 @@ func runE2B(ctx context.Context, o options) error { return fmt.Errorf("sandboxID %q is not DNS-label safe", id) } - // 3. List and Get. var listed []map[string]any if err := e.do(ctx, http.MethodGet, "/sandboxes", nil, &listed); err != nil { return fmt.Errorf("list: %w", err) @@ -349,7 +331,6 @@ func runE2B(ctx context.Context, o options) error { } stepf("get", "%s is in the read view", id) - // 4. Metrics. var metrics []map[string]any if err := e.do(ctx, http.MethodGet, "/sandboxes/"+id+"/metrics", nil, &metrics); err != nil { return fmt.Errorf("metrics: %w", err) @@ -358,7 +339,6 @@ func runE2B(ctx context.Context, o options) error { stepf("metrics", "cpuCount=%v memTotal=%v", metrics[0]["cpuCount"], metrics[0]["memTotal"]) } - // 5. Snapshot, then list snapshots. var snap map[string]any if err := e.do(ctx, http.MethodPost, "/sandboxes/"+id+"/snapshots", map[string]any{"name": "example-e2b-snap"}, &snap); err != nil { @@ -372,7 +352,6 @@ func runE2B(ctx context.Context, o options) error { } stepf("snapshots", "%d checkpoint(s) fleet-wide", len(snaps)) - // 6. Fork — one result per child, each a new sandbox. var forks []map[string]any if err := e.do(ctx, http.MethodPost, "/sandboxes/"+id+"/fork", map[string]any{"count": 2, "timeout": 600}, &forks); err != nil { @@ -380,8 +359,6 @@ func runE2B(ctx context.Context, o options) error { } stepf("fork", "%d child sandbox(es)", len(forks)) - // 7. Pause, and prove the already-paused contract: the SDK reads 409 as - // "was already paused" and returns false rather than raising. if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/pause", nil); err != nil { return err } else if code != http.StatusNoContent { @@ -396,8 +373,6 @@ func runE2B(ctx context.Context, o options) error { } stepf("pause", "409 on repeat — the already-paused contract holds") - // 8. Connect is the SDK's resume: 201 when it actually restored a paused - // sandbox, 200 when it was already running. if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/connect", map[string]any{"timeout": 600}); err != nil { return err @@ -414,7 +389,6 @@ func runE2B(ctx context.Context, o options) error { } stepf("connect", "200 — already running, no restore") - // 9. setTimeout and the keepalive. if code, err := e.status(ctx, http.MethodPost, "/sandboxes/"+id+"/timeout", map[string]any{"timeout": 900}); err != nil { return err @@ -431,7 +405,6 @@ func runE2B(ctx context.Context, o options) error { } stepf("refreshes", "keepalive accepted") - // 10. Delete. if o.keep { stepf("delete", "skipped (-keep)") return nil @@ -468,24 +441,13 @@ func (e *e2bClient) health(ctx context.Context) error { // waitVisible polls until the read view publishes the sandbox — the same // eventual consistency the Kubernetes surface has, for the same reason. func (e *e2bClient) waitVisible(ctx context.Context, id string) error { - deadline := time.Now().Add(visibilityTimeout) - for { + return pollVisible(ctx, "sandbox "+id, func() (bool, error) { code, err := e.status(ctx, http.MethodGet, "/sandboxes/"+id, nil) if err != nil { - return err - } - if code == http.StatusOK { - return nil + return false, err } - if time.Now().After(deadline) { - return fmt.Errorf("sandbox %s not visible within %s", id, visibilityTimeout) - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(3 * time.Second): - } - } + return code == http.StatusOK, nil + }) } func (e *e2bClient) request(ctx context.Context, method, path string, body any) (*http.Request, error) { @@ -551,6 +513,28 @@ func (e *e2bClient) status(ctx context.Context, method, path string, body any) ( return resp.StatusCode, nil } +// pollVisible retries probe on a 3s tick until the subject is visible or visibilityTimeout elapses. +func pollVisible(ctx context.Context, subject string, probe func() (bool, error)) error { + deadline := time.Now().Add(visibilityTimeout) + for { + visible, err := probe() + if err != nil { + return err + } + if visible { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("%s not visible within %s", subject, visibilityTimeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(3 * time.Second): + } + } +} + func section(name string) { fmt.Printf("\n=== %s ===\n", name) } func stepf(verb, format string, args ...any) { diff --git a/extensions/controllers/sandboxclaim_controller.go b/extensions/controllers/sandboxclaim_controller.go index ad6565b..c19ed1b 100644 --- a/extensions/controllers/sandboxclaim_controller.go +++ b/extensions/controllers/sandboxclaim_controller.go @@ -146,8 +146,6 @@ type SandboxClaimReconciler struct { //+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch;update //+kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. func (r *SandboxClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) logger.V(1).Info("Start of Reconcile loop for SandboxClaim", "request", req.NamespacedName) @@ -296,8 +294,6 @@ func (r *SandboxClaimReconciler) SetupWithManager(mgr ctrl.Manager, concurrentWo handler.EnqueueRequestsFromMapFunc(r.mapWarmPoolToClaims), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). - // TODO: Keep a lightweight SandboxTemplate -> claims map watch to promptly reconcile - // claims when a missing template is created, instead of relying on the 1-minute fallback. WithOptions(controller.Options{MaxConcurrentReconciles: concurrentWorkers}). Complete(r) } @@ -329,12 +325,6 @@ func (r *SandboxClaimReconciler) resultFor(ctx context.Context, claim *extension return ctrl.Result{RequeueAfter: soonerRequeue(result.RequeueAfter, time.Minute)}, true } - // Adoption patched the sandbox to us but the informer cache may still show the - // warm-pool owner. Requeue without an error: an error routes through the - // exponential failure rate limiter, and because this same retry recurs each pass - // until the cache catches up the backoff compounds (#1107). A nil error lets the - // workqueue Forget the key. Status is intentionally not finalized on this pass, - // preserving the duplicate-adoption protection during cache lag. if errors.Is(reconcileErr, errAdoptionTriggeredRetry) { logger.V(4).Info("Adoption triggered; requeueing to let cache converge", "claim", claim.Name, "error", reconcileErr) return ctrl.Result{RequeueAfter: soonerRequeue(result.RequeueAfter, adoptionCacheLagRequeueDelay)}, true @@ -475,7 +465,7 @@ func (r *SandboxClaimReconciler) reconcileActive(ctx context.Context, claim *ext } // Fast path: try to find existing or adopt from warm pool before template lookup. - sandbox, err := r.getOrCreateSandbox(ctx, claim, nil) + sandbox, err := r.getOrCreateSandbox(ctx, claim) logger.V(1).Info("getOrCreateSandbox result", "sandboxFound", sandbox != nil, "err", err, "claim", claim.Name) if err != nil { return nil, err @@ -509,17 +499,15 @@ func (r *SandboxClaimReconciler) reconcileExpired(ctx context.Context, claim *ex sandbox := &v1beta1.Sandbox{} if err := r.Get(ctx, client.ObjectKey{Namespace: claim.Namespace, Name: statusName}, sandbox); err != nil { if k8errors.IsNotFound(err) { - return nil, nil // Sandbox is gone, life is good. + return nil, nil } return nil, err } - // Verify ownership before delete action if !metav1.IsControlledBy(sandbox, claim) { logger.Info("Skipping deletion: Sandbox is not controlled by this claim", "sandbox", sandbox.Name, "claim", claim.Name) return nil, fmt.Errorf("%w: sandbox %q is not owned by claim %q", ErrSandboxNotOwned, sandbox.Name, claim.Name) } - // Sandbox exists, delete it. if sandbox.DeletionTimestamp.IsZero() { logger.Info("Deleting Sandbox because Claim expired (Policy=Retain)", "sandbox", sandbox.Name, "claim", claim.Name) if err := r.Delete(ctx, sandbox); err != nil { @@ -756,8 +744,6 @@ func (r *SandboxClaimReconciler) tryAdopt(ctx context.Context, claim *extensions } logger.Info("Successfully adopted sandbox from warm pool", "sandbox", adopted.Name, "claim", claim.Name) - // Recorded so a later pass still seeing the stale warm-pool-owned view waits - // out the bounded requeue instead of re-sending the adoption patch. r.triggeredAdoptions.Store( types.NamespacedName{Name: claim.Name, Namespace: claim.Namespace}, triggeredAdoptionEntry{uid: claim.UID, sandbox: adopted.Name}, @@ -810,11 +796,9 @@ func (r *SandboxClaimReconciler) completeAdoption(ctx context.Context, claim *ex adopted.Annotations[asmetrics.TraceContextAnnotation] = traceContext } - // Propagate claim identity labels for discovery and NetworkPolicy targeting. adopted.Labels = ensureClaimIdentityLabels(adopted.Labels, claim) adopted.Spec.PodTemplate.ObjectMeta.Labels = ensureClaimIdentityLabels(adopted.Spec.PodTemplate.ObjectMeta.Labels, claim) - // Resolve the template hash and metadata used by reconcileActive. template, templateErr := r.getTemplate(ctx, claim) if templateHash == "" && template != nil { templateHash = SandboxTemplateRefHash(template.Name) @@ -838,8 +822,6 @@ func (r *SandboxClaimReconciler) completeAdoption(ctx context.Context, claim *ex mergedMeta.Labels = make(map[string]string) } mergedMeta.Labels[extensionsv1beta1.SandboxIDLabel] = string(claim.UID) - // A claim without created-by must clear it, so the label is never inherited - // from whoever held this pre-warmed sandbox before. setOrDeleteLabel(mergedMeta.Labels, v1beta1.CreatedByLabel, claim.Labels[v1beta1.CreatedByLabel]) adopted.Spec.PodTemplate.ObjectMeta = mergedMeta } @@ -942,7 +924,6 @@ func (r *SandboxClaimReconciler) mergePodMetadata(templateMeta *v1beta1.PodMetad } } - // Merge labels if len(claimMeta.Labels) > 0 { if templateMeta.Labels == nil { templateMeta.Labels = make(map[string]string) @@ -950,7 +931,6 @@ func (r *SandboxClaimReconciler) mergePodMetadata(templateMeta *v1beta1.PodMetad maps.Copy(templateMeta.Labels, claimMeta.Labels) } - // Merge annotations if len(claimMeta.Annotations) > 0 { if templateMeta.Annotations == nil { templateMeta.Annotations = make(map[string]string) @@ -1005,7 +985,6 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten Name: claim.Name, } - // Propagate the trace context annotation to the Sandbox resource if sandbox.Annotations == nil { sandbox.Annotations = make(map[string]string) } @@ -1039,10 +1018,6 @@ func (r *SandboxClaimReconciler) createSandbox(ctx context.Context, claim *exten } } - // Propagate claim identity labels for discovery and NetworkPolicy targeting. - // Fork extension: also write SandboxIDLabel onto the top-level Sandbox metadata - // (KEP-0174 only propagates to pod template labels; platform's informer reads - // Sandbox.metadata.labels). templateHash := SandboxTemplateRefHash(template.Name) sandbox.Labels = ensureClaimIdentityLabels(sandbox.Labels, claim) sandbox.Labels[v1beta1.SandboxLaunchTypeLabel] = v1beta1.SandboxLaunchTypeCold @@ -1146,7 +1121,7 @@ func (r *SandboxClaimReconciler) injectClaimEnv(logger logr.Logger, claim *exten return nil } -func (r *SandboxClaimReconciler) getOrCreateSandbox(ctx context.Context, claim *extensionsv1beta1.SandboxClaim, _ *extensionsv1beta1.SandboxTemplate) (*v1beta1.Sandbox, error) { +func (r *SandboxClaimReconciler) getOrCreateSandbox(ctx context.Context, claim *extensionsv1beta1.SandboxClaim) (*v1beta1.Sandbox, error) { logger := log.FromContext(ctx) logger.V(1).Info("Executing getOrCreateSandbox", "claim", claim.Name) @@ -1259,8 +1234,6 @@ func (r *SandboxClaimReconciler) completePendingAdoption(ctx context.Context, cl return nil } - // The adoption patch is idempotent, but re-sending it while the cache lags adds - // nothing; wait out the bounded requeue instead. adoptionKey := types.NamespacedName{Name: claim.Name, Namespace: claim.Namespace} if prev, ok := r.triggeredAdoptions.Load(adoptionKey); ok && prev.uid == claim.UID && prev.sandbox == sbName { logger.V(4).Info("Adoption already triggered, waiting for cache to converge", "sandbox", sbName, "claim", claim.Name) @@ -1276,9 +1249,6 @@ func (r *SandboxClaimReconciler) completePendingAdoption(ctx context.Context, cl } r.triggeredAdoptions.Store(adoptionKey, triggeredAdoptionEntry{uid: claim.UID, sandbox: sbName}) - // completeAdoption patched our controllerRef and the Warm label. Requeue via the - // sentinel so this does not route through the exponential failure rate limiter, - // whose compounding backoff ballooned adoption tail latency (#1107). logger.Info("Triggered adoption completion for sandbox, requeueing", "sandbox", sbName, "claim", claim.Name) return fmt.Errorf("%w: sandbox %s", errAdoptionTriggeredRetry, sbName) } @@ -1489,12 +1459,7 @@ func (r *SandboxClaimReconciler) recordSandboxCreationLatency(sandbox *v1beta1.S } // recordCreationLatencyMetric detects and records transitions to Ready state. -func (r *SandboxClaimReconciler) recordCreationLatencyMetric( - ctx context.Context, - claim *extensionsv1beta1.SandboxClaim, - oldStatus *extensionsv1beta1.SandboxClaimStatus, - sandbox *v1beta1.Sandbox, -) { +func (r *SandboxClaimReconciler) recordCreationLatencyMetric(ctx context.Context, claim *extensionsv1beta1.SandboxClaim, oldStatus *extensionsv1beta1.SandboxClaimStatus, sandbox *v1beta1.Sandbox) { logger := log.FromContext(ctx) newStatus := &claim.Status @@ -1817,8 +1782,6 @@ func readyFailure(claim *extensionsv1beta1.SandboxClaim, err error) failure { case errors.Is(err, ErrWarmPoolNotFound): return failure{"WarmPoolNotFound", fmt.Sprintf("SandboxWarmPool %q not found", claim.Spec.WarmPoolRef.Name)} case errors.Is(err, errAdoptionTriggeredRetry): - // Benign: adoption was patched and we are only waiting for the informer - // cache to converge before finalizing. return failure{"AdoptionPending", "Warm-pool sandbox adoption triggered; waiting for cache to converge"} case errors.Is(err, ErrInvalidMetadata): return failure{reasonInvalidMetadata, err.Error()} @@ -1844,13 +1807,7 @@ func ensureClaimIdentityLabels(labels map[string]string, claim *extensionsv1beta labels = make(map[string]string) } labels[extensionsv1beta1.SandboxIDLabel] = string(claim.UID) - // Propagate created-by label from the claim if present. If absent, explicitly - // delete it to synchronize removal or prevent stale propagation from warm sandboxes. - if val, ok := claim.Labels[v1beta1.CreatedByLabel]; ok && val != "" { - labels[v1beta1.CreatedByLabel] = val - } else { - delete(labels, v1beta1.CreatedByLabel) - } + setOrDeleteLabel(labels, v1beta1.CreatedByLabel, claim.Labels[v1beta1.CreatedByLabel]) return labels } @@ -1905,11 +1862,7 @@ func domainInList(domain string, list []string) bool { }) } -func mergeVolumeClaimTemplates( - templateVCTs []v1beta1.PersistentVolumeClaimTemplate, - claimVCTs []v1beta1.PersistentVolumeClaimTemplate, - policy extensionsv1beta1.VolumeClaimTemplatesPolicy, -) ([]v1beta1.PersistentVolumeClaimTemplate, error) { +func mergeVolumeClaimTemplates(templateVCTs, claimVCTs []v1beta1.PersistentVolumeClaimTemplate, policy extensionsv1beta1.VolumeClaimTemplatesPolicy) ([]v1beta1.PersistentVolumeClaimTemplate, error) { if err := validateVolumeClaimTemplates(templateVCTs); err != nil { return nil, fmt.Errorf("template: %w", err) } diff --git a/extensions/controllers/sandboxtemplate_controller.go b/extensions/controllers/sandboxtemplate_controller.go index 9cf4ec0..049d2f9 100644 --- a/extensions/controllers/sandboxtemplate_controller.go +++ b/extensions/controllers/sandboxtemplate_controller.go @@ -237,37 +237,33 @@ func buildDefaultNetworkPolicySpec(templateName, routerNamespace string) network networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress, }, - // 1. INGRESS: Allow traffic only from the Sandbox Router Ingress: []networkingv1.NetworkPolicyIngressRule{ { From: peers, }, }, - // 2. EGRESS: Secure Default Configuration Egress: []networkingv1.NetworkPolicyEgressRule{ - // Public Internet Access (Strict Isolation) - // This rule allows all ports to PUBLIC IPs, but explicitly blocks private LAN ranges. - // NOTE: This intentionally blocks internal cluster DNS (CoreDNS) by default to prevent - // agents from probing for service discovery and leaking internal service names. + // Blocking the private ranges also blocks cluster DNS, so agents cannot + // probe service discovery and leak internal service names. { To: []networkingv1.NetworkPolicyPeer{ { IPBlock: &networkingv1.IPBlock{ CIDR: "0.0.0.0/0", Except: []string{ - "10.0.0.0/8", // Block Private Class A (Cluster/VPC Network) - "172.16.0.0/12", // Block Private Class B - "192.168.0.0/16", // Block Private Class C - "169.254.0.0/16", // Block Link-Local (Metadata Server) + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "169.254.0.0/16", // metadata server }, }, }, { IPBlock: &networkingv1.IPBlock{ - CIDR: "::/0", // IPv6 Catch-all + CIDR: "::/0", Except: []string{ - "fc00::/7", // Block IPv6 Unique Local Addresses (Internal) - "fe80::/10", // Block IPv6 Link-Local + "fc00::/7", + "fe80::/10", }, }, }, diff --git a/extensions/controllers/sandboxwarmpool_controller.go b/extensions/controllers/sandboxwarmpool_controller.go index 55b4787..9705a63 100644 --- a/extensions/controllers/sandboxwarmpool_controller.go +++ b/extensions/controllers/sandboxwarmpool_controller.go @@ -58,6 +58,14 @@ const ( sandboxWarmPoolLabelIndex = ".metadata.labels[" + warmPoolSandboxLabel + "]" ) +// staleCheck is the resolved template a pool member is vetted against, with the memo of hashes already compared. +type staleCheck struct { + template *extensionsv1beta1.SandboxTemplate + refHash string + blueprintHash string + vetted map[string]bool +} + // SandboxWarmPoolReconciler reconciles a SandboxWarmPool object. type SandboxWarmPoolReconciler struct { client.Client @@ -83,7 +91,6 @@ type SandboxWarmPoolReconciler struct { //+kubebuilder:rbac:groups=extensions.agents.x-k8s.io,resources=sandboxwarmpools/status,verbs=get;update;patch //+kubebuilder:rbac:groups=agents.x-k8s.io,resources=sandboxes,verbs=get;list;watch;create;update;patch;delete -// Reconcile implements the reconciliation loop for SandboxWarmPool. func (r *SandboxWarmPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -148,9 +155,6 @@ func (r *SandboxWarmPoolReconciler) SetupWithManager(mgr ctrl.Manager, concurren func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool *extensionsv1beta1.SandboxWarmPool) (time.Duration, error) { logger := log.FromContext(ctx) - // In the L3 writable-aggregation design warm capacity lives per-node in - // sandboxd, not as Sandbox CRs, and creating CRs would fight the aggregated - // node-local claim path. When disabled, report status without any create/delete. if r.DisableSandboxCRManagement { return 0, r.reconcilePoolStatusOnly(ctx, warmPool) } @@ -232,12 +236,11 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool sandboxesToCreate := min(desiredReplicas-currentReplicas, maxBatchSize) logger.Info("Creating new pool sandboxes", "count", sandboxesToCreate) - sandboxCR, err := r.buildSandboxCR(ctx, warmPool, poolNameHash, template, currentSandboxBlueprintHash) + sandboxCR, err := r.buildSandboxCR(ctx, warmPool, template, currentSandboxBlueprintHash) if err != nil { logger.Error(err, "Failed to build sandbox CR blueprint") allErrors = errors.Join(allErrors, err) } else { - // Parallel sandbox creation with adaptive slow-start batching (starts with 1 and doubles on success) _, createErr := slowStartBatch(ctx, int(sandboxesToCreate), 1, func(_ int) error { return r.createPoolSandbox(ctx, warmPool, sandboxCR) }) @@ -259,15 +262,14 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool bReady := isSandboxReady(b) if aReady != bReady { if aReady { - return 1 // a ready, b not ready -> b first (delete unready first) + return 1 } - return -1 // b ready, a not ready -> a first + return -1 } - return b.CreationTimestamp.Compare(a.CreationTimestamp.Time) // newest first + return b.CreationTimestamp.Compare(a.CreationTimestamp.Time) }) toDeleteCount := min(sandboxesToDelete, int32(len(activeSandboxes))) - // Parallel sandbox deletion with adaptive slow-start batching (starts with 1 and doubles on success) _, deleteErr := slowStartBatch(ctx, int(toDeleteCount), 1, func(idx int) error { return r.deletePoolSandbox(ctx, activeSandboxes[idx]) }) @@ -284,11 +286,6 @@ func (r *SandboxWarmPoolReconciler) reconcilePool(ctx context.Context, warmPool return stuckRecheck, allErrors } -// reconcilePoolStatusOnly reports pool status without creating or deleting any -// Sandbox CRs. It is the reconcile used when DisableSandboxCRManagement is set: -// warm capacity is driven per-node by sandboxd, so the controller only surfaces -// the current CR-backed member count (typically zero in that mode) and never -// mutates the aggregated node-local claim path. func (r *SandboxWarmPoolReconciler) reconcilePoolStatusOnly(ctx context.Context, warmPool *extensionsv1beta1.SandboxWarmPool) error { poolNameHash := hash.Name(warmPool.Name) labelSelector := labels.SelectorFromSet(labels.Set{warmPoolSandboxLabel: poolNameHash}) @@ -331,10 +328,9 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w var activeSandboxes []*sandboxv1beta1.Sandbox var allErrors error - vettedHashes := make(map[string]bool) - var currentTemplateRefHash string + check := staleCheck{template: template, blueprintHash: currentSandboxBlueprintHash, vetted: make(map[string]bool)} if template != nil { - currentTemplateRefHash = SandboxTemplateRefHash(template.Name) + check.refHash = SandboxTemplateRefHash(template.Name) } var updateStrategyType extensionsv1beta1.SandboxWarmPoolUpdateStrategyType @@ -369,7 +365,7 @@ func (r *SandboxWarmPoolReconciler) filterActiveSandboxes(ctx context.Context, w } if tmplErr == nil && (updateStrategy == extensionsv1beta1.RecreateSandboxWarmPoolUpdateStrategyType || isOrphan) { - if r.isSandboxStale(ctx, sb, template, currentTemplateRefHash, currentSandboxBlueprintHash, vettedHashes) { + if r.isSandboxStale(ctx, sb, check) { logger.Info("Deleting stale sandbox", "sandbox", sb.Name, "isOrphan", isOrphan) if err := r.Delete(ctx, sb); err != nil { logger.Error(err, "Failed to delete stale sandbox", "sandbox", sb.Name) @@ -423,13 +419,8 @@ func (r *SandboxWarmPoolReconciler) fetchTemplateAndHash(ctx context.Context, wa } // buildSandboxCR constructs the base Sandbox CR (with pod template and volume claim templates) for the warm pool. -func (r *SandboxWarmPoolReconciler) buildSandboxCR( - ctx context.Context, - warmPool *extensionsv1beta1.SandboxWarmPool, - poolNameHash string, - template *extensionsv1beta1.SandboxTemplate, - currentSandboxBlueprintHash string, -) (*sandboxv1beta1.Sandbox, error) { +func (r *SandboxWarmPoolReconciler) buildSandboxCR(ctx context.Context, warmPool *extensionsv1beta1.SandboxWarmPool, template *extensionsv1beta1.SandboxTemplate, currentSandboxBlueprintHash string) (*sandboxv1beta1.Sandbox, error) { + poolNameHash := hash.Name(warmPool.Name) sandboxLabels := map[string]string{ warmPoolSandboxLabel: poolNameHash, sandboxTemplateRefHash: SandboxTemplateRefHash(warmPool.Spec.TemplateRef.Name), @@ -546,47 +537,40 @@ func (r *SandboxWarmPoolReconciler) getTemplate(ctx context.Context, warmPool *e // isSandboxStale checks if the sandbox version matches the current template. // It uses a cache (vettedHashes) to avoid repeated expensive DeepEqual calls // for sandboxes with the same hash. -func (r *SandboxWarmPoolReconciler) isSandboxStale( - ctx context.Context, - sandbox *sandboxv1beta1.Sandbox, - template *extensionsv1beta1.SandboxTemplate, - currentTemplateRefHash string, - currentSandboxBlueprintHash string, - vettedHashes map[string]bool, -) bool { +func (r *SandboxWarmPoolReconciler) isSandboxStale(ctx context.Context, sandbox *sandboxv1beta1.Sandbox, check staleCheck) bool { sandboxHash := sandbox.Labels[sandboxv1beta1.SandboxTemplateHashLabel] - if sandbox.Labels[sandboxTemplateRefHash] != currentTemplateRefHash { + if sandbox.Labels[sandboxTemplateRefHash] != check.refHash { return true } controllerRef := metav1.GetControllerOf(sandbox) isOrphan := controllerRef == nil if isOrphan { - return !r.compareSandboxBlueprint(template, &sandbox.Spec.SandboxBlueprint) + return !r.compareSandboxBlueprint(check.template, &sandbox.Spec.SandboxBlueprint) } - if sandboxHash != "" && sandboxHash == currentSandboxBlueprintHash { + if sandboxHash != "" && sandboxHash == check.blueprintHash { return false } // A marshal failure leaves the hash empty; treating that as stale would // mass-delete the pool. - if currentSandboxBlueprintHash == "" { - log.FromContext(ctx).Error(nil, "currentSandboxBlueprintHash is empty, skipping staleness check", "sandbox", sandbox.Name) + if check.blueprintHash == "" { + log.FromContext(ctx).Error(nil, "blueprint hash is empty, skipping staleness check", "sandbox", sandbox.Name) return false } if sandboxHash != "" { - if isStale, found := vettedHashes[sandboxHash]; found { + if isStale, found := check.vetted[sandboxHash]; found { return isStale } } - isStale := !r.compareSandboxBlueprint(template, &sandbox.Spec.SandboxBlueprint) + isStale := !r.compareSandboxBlueprint(check.template, &sandbox.Spec.SandboxBlueprint) if sandboxHash != "" { - vettedHashes[sandboxHash] = isStale + check.vetted[sandboxHash] = isStale } return isStale diff --git a/extensions/controllers/sandboxwarmpool_controller_test.go b/extensions/controllers/sandboxwarmpool_controller_test.go index 9eeecef..dd990ac 100644 --- a/extensions/controllers/sandboxwarmpool_controller_test.go +++ b/extensions/controllers/sandboxwarmpool_controller_test.go @@ -1287,7 +1287,8 @@ func TestIsSandboxStale_OrphanedSandboxVetting(t *testing.T) { Spec: sandboxv1beta1.SandboxSpec{SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{Spec: *spoofedSpec}}}, } - isStaleSpoofed := r.isSandboxStale(ctx, spoofedOrphan, template, SandboxTemplateRefHash(template.Name), currentSandboxBlueprintHash, vettedHashes) + check := staleCheck{template: template, refHash: SandboxTemplateRefHash(template.Name), blueprintHash: currentSandboxBlueprintHash, vetted: vettedHashes} + isStaleSpoofed := r.isSandboxStale(ctx, spoofedOrphan, check) require.True(t, isStaleSpoofed, "Orphaned sandbox with spoofed hash but modified PodSpec should be stale") genuineSpec := template.Spec.PodTemplate.Spec.DeepCopy() @@ -1304,7 +1305,7 @@ func TestIsSandboxStale_OrphanedSandboxVetting(t *testing.T) { Spec: sandboxv1beta1.SandboxSpec{SandboxBlueprint: sandboxv1beta1.SandboxBlueprint{PodTemplate: sandboxv1beta1.PodTemplate{Spec: *genuineSpec}}}, } - isStaleGenuine := r.isSandboxStale(ctx, genuineOrphan, template, SandboxTemplateRefHash(template.Name), currentSandboxBlueprintHash, vettedHashes) + isStaleGenuine := r.isSandboxStale(ctx, genuineOrphan, check) require.False(t, isStaleGenuine, "Orphaned sandbox with genuine fully vetted PodSpec should be fresh") } @@ -2009,7 +2010,7 @@ func TestNewPoolSandboxesCarryOnlyTheRenamedHashLabel(t *testing.T) { Name: "p", Namespace: "default", Spec: extensionsv1beta1.SandboxWarmPoolSpec{TemplateRef: extensionsv1beta1.SandboxTemplateRef{Name: "test-template"}}, } - sb, err := r.buildSandboxCR(t.Context(), warmPool, "hash", createTemplate("default"), "bph") + sb, err := r.buildSandboxCR(t.Context(), warmPool, createTemplate("default"), "bph") if err != nil { t.Fatal(err) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 3b6422d..055904e 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -12,22 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive package metrics import ( + "fmt" "time" "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/cocoonstack/sandbox-operator/internal/version" ) const ( - LaunchTypeWarm = "warm" // Pod from a SandboxWarmPool - LaunchTypeCold = "cold" // Pod not from a SandboxWarmPool - LaunchTypeUnknown = "unknown" // Used when Sandbox is nil during failure + LaunchTypeWarm = "warm" + LaunchTypeCold = "cold" + LaunchTypeUnknown = "unknown" // Sandbox is nil during failure // ObservabilityAnnotation is the annotation key for the time the controller first observed the claim. ObservabilityAnnotation = "agents.x-k8s.io/controller-first-observed-at" @@ -120,21 +119,25 @@ var ( }, func() float64 { return 1 }, ) - - // Registration rides on package variable initialization so no caller can observe an unregistered collector. - _ = func() bool { - metrics.Registry.MustRegister( - ClaimStartupLatency, - ClaimControllerStartupLatency, - SandboxCreationLatency, - SandboxClaimCreationTotal, - WarmPoolSandboxCreatedTotal, - BuildInfo, - ) - return true - }() ) +// Register adds the operator's collectors to r. A binary calls it once. +func Register(r prometheus.Registerer) error { + for _, c := range []prometheus.Collector{ + ClaimStartupLatency, + ClaimControllerStartupLatency, + SandboxCreationLatency, + SandboxClaimCreationTotal, + WarmPoolSandboxCreatedTotal, + BuildInfo, + } { + if err := r.Register(c); err != nil { + return fmt.Errorf("register operator metrics: %w", err) + } + } + return nil +} + // IncWarmPoolSandboxCreated counts one Sandbox created by the warm-pool controller. func IncWarmPoolSandboxCreated(namespace, warmPoolName string) { WarmPoolSandboxCreatedTotal.WithLabelValues(namespace, warmPoolName).Inc() diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index fb81c5f..7574a06 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive package metrics import ( @@ -21,6 +20,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/propagation" @@ -117,6 +117,27 @@ func TestBuildInfo(t *testing.T) { } } +func TestRegisterIsExplicitAndIdempotentlyRefused(t *testing.T) { + registry := prometheus.NewRegistry() + count, err := testutil.GatherAndCount(registry, "agent_sandbox_build_info") + if err != nil { + t.Fatalf("gather: %v", err) + } + if count != 0 { + t.Fatalf("collectors registered before Register was called, count = %d", count) + } + + if err := Register(registry); err != nil { + t.Fatalf("Register: %v", err) + } + if count, err = testutil.GatherAndCount(registry, "agent_sandbox_build_info"); err != nil || count != 1 { + t.Errorf("after Register: count = %d, err = %v", count, err) + } + if err := Register(registry); err == nil { + t.Error("a second Register must report the duplicate registration") + } +} + func TestStartSpanEndFuncEndsSpan(t *testing.T) { exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) diff --git a/internal/metrics/sandbox_collector.go b/internal/metrics/sandbox_collector.go index b15b5c3..cb5e87f 100644 --- a/internal/metrics/sandbox_collector.go +++ b/internal/metrics/sandbox_collector.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive package metrics import ( diff --git a/internal/metrics/sandbox_collector_test.go b/internal/metrics/sandbox_collector_test.go index 2b66a9c..7f845fd 100644 --- a/internal/metrics/sandbox_collector_test.go +++ b/internal/metrics/sandbox_collector_test.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive package metrics import ( diff --git a/internal/metrics/tracing.go b/internal/metrics/tracing.go index 4e2542f..3858471 100644 --- a/internal/metrics/tracing.go +++ b/internal/metrics/tracing.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive package metrics import ( diff --git a/internal/version/version_test.go b/internal/version/version_test.go index 8f6dd20..553f0e0 100644 --- a/internal/version/version_test.go +++ b/internal/version/version_test.go @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive package version import ( @@ -81,6 +80,15 @@ func TestInfoString(t *testing.T) { } } +func TestPrintRendersProgramAndVersion(t *testing.T) { + out := Print("sandbox-operator") + for _, want := range []string{"sandbox-operator, version", "build date:", "platform:"} { + if !strings.Contains(out, want) { + t.Errorf("Print output missing %q, got: %s", want, out) + } + } +} + func TestInfoPrinted(t *testing.T) { platform := fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) diff --git a/pkg/e2bcompat/lifecycle.go b/pkg/e2bcompat/lifecycle.go index debe150..042e865 100644 --- a/pkg/e2bcompat/lifecycle.go +++ b/pkg/e2bcompat/lifecycle.go @@ -27,8 +27,7 @@ const maxNodeConcurrency = 16 // returns false rather than raising — so that state is reported, not retried. func (s *Server) pauseSandbox(w http.ResponseWriter, r *http.Request) { var req SandboxPauseRequest - if err := decodeOptional(w, r, &req); err != nil { - writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + if !decodeOptionalBody(w, r, &req) { return } id := r.PathValue("sandboxID") @@ -65,8 +64,7 @@ func (s *Server) pauseSandbox(w http.ResponseWriter, r *http.Request) { // preceded it. func (s *Server) connectSandbox(w http.ResponseWriter, r *http.Request) { var req ConnectSandbox - if err := decodeOptional(w, r, &req); err != nil { - writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + if !decodeOptionalBody(w, r, &req) { return } id := r.PathValue("sandboxID") @@ -103,8 +101,7 @@ func (s *Server) connectSandbox(w http.ResponseWriter, r *http.Request) { // it rejects the request outright. func (s *Server) forkSandbox(w http.ResponseWriter, r *http.Request) { var req SandboxForkRequest - if err := decodeOptional(w, r, &req); err != nil { - writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + if !decodeOptionalBody(w, r, &req) { return } count := int32(1) @@ -155,8 +152,7 @@ func (s *Server) forkSandbox(w http.ResponseWriter, r *http.Request) { // can branch from. The source keeps running. func (s *Server) createSnapshot(w http.ResponseWriter, r *http.Request) { var req SandboxSnapshotRequest - if err := decodeOptional(w, r, &req); err != nil { - writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + if !decodeOptionalBody(w, r, &req) { return } id := r.PathValue("sandboxID") @@ -363,15 +359,24 @@ func (s *Server) isPaused(ctx context.Context, sb *sandboxv1beta1.Sandbox) bool return sb.Labels[scale.PhaseLabel] == phaseHibernated } -// decodeOptional decodes a JSON body that the schema allows to be absent. An -// empty body leaves the target at its zero value rather than failing, which is -// what pause and fork require. -func decodeOptional(w http.ResponseWriter, r *http.Request, out any) error { +func decodeBody(w http.ResponseWriter, r *http.Request, out any) bool { + return reportBadBody(w, json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(out)) +} + +func decodeOptionalBody(w http.ResponseWriter, r *http.Request, out any) bool { err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(out) if errors.Is(err, io.EOF) { - return nil + return true + } + return reportBadBody(w, err) +} + +func reportBadBody(w http.ResponseWriter, err error) bool { + if err != nil { + writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + return false } - return err + return true } // claimIDOf reports the node-local claim id the store's verbs address. diff --git a/pkg/e2bcompat/server.go b/pkg/e2bcompat/server.go index d47a2c9..91e2250 100644 --- a/pkg/e2bcompat/server.go +++ b/pkg/e2bcompat/server.go @@ -21,7 +21,6 @@ package e2bcompat import ( "crypto/subtle" - "encoding/json" "errors" "fmt" "net/http" @@ -190,8 +189,7 @@ func (s *Server) validKey(presented string) bool { // same node-local claim the aggregated apiserver's Create performs. func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request) { var req NewSandbox - if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + if !decodeBody(w, r, &req) { return } if strings.TrimSpace(req.TemplateID) == "" { @@ -289,8 +287,7 @@ func (s *Server) deleteSandbox(w http.ResponseWriter, r *http.Request) { // it does not silently claim to have extended a deadline it cannot move. func (s *Server) setTimeout(w http.ResponseWriter, r *http.Request) { var req SandboxTimeoutRequest - if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(&req); err != nil { - writeError(w, http.StatusBadRequest, fmt.Sprintf("invalid request body: %v", err)) + if !decodeBody(w, r, &req) { return } if req.Timeout < 0 { diff --git a/pkg/podruntime/cocoon.go b/pkg/podruntime/cocoon.go index cf9588f..98d3dc2 100644 --- a/pkg/podruntime/cocoon.go +++ b/pkg/podruntime/cocoon.go @@ -29,7 +29,6 @@ const ( // opts into a virtual-node contract. DefaultMode = ModeStandard - // vk-cocoon (cocoon MicroVM) virtual-node contract. vkProviderTaintKey = "virtual-kubelet.io/provider" vkNodeLabelKey = "node.kubernetes.io/instance-type" vkNodeLabelValue = "virtual-node" diff --git a/pkg/sandboxd/client.go b/pkg/sandboxd/client.go index bc1eeed..13ee180 100644 --- a/pkg/sandboxd/client.go +++ b/pkg/sandboxd/client.go @@ -14,6 +14,7 @@ import ( "io" "net/http" "net/url" + "slices" "strings" "time" ) @@ -207,8 +208,12 @@ func (c *Client) Release(ctx context.Context, id, token string) error { if id == "" { return fmt.Errorf("sandboxd: release requires a sandbox id") } - u := c.baseURL + "/v1/sandboxes/" + url.PathEscape(id) + "/release" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil) + return c.sendNoBody(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/release", token, "release", http.StatusNoContent, http.StatusNotFound) +} + +// sendNoBody performs a body-less request authenticated with token, accepting the statuses in ok. +func (c *Client) sendNoBody(ctx context.Context, method, path, token, op string, ok ...int) error { + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, nil) if err != nil { return err } @@ -216,16 +221,14 @@ func (c *Client) Release(ctx context.Context, id, token string) error { resp, err := c.hc.Do(req) if err != nil { - return fmt.Errorf("sandboxd: release: %w", err) + return fmt.Errorf("sandboxd: %s: %w", op, err) } defer drainAndClose(resp) - switch resp.StatusCode { - case http.StatusNoContent, http.StatusNotFound: - return nil - default: + if !slices.Contains(ok, resp.StatusCode) { return statusError(resp) } + return nil } func (c *Client) authenticate(req *http.Request, token string) { diff --git a/pkg/sandboxd/lifecycle.go b/pkg/sandboxd/lifecycle.go index bc38c82..e62e39d 100644 --- a/pkg/sandboxd/lifecycle.go +++ b/pkg/sandboxd/lifecycle.go @@ -16,10 +16,6 @@ import ( // design projects. const maxReplyBytes = 16 << 20 -// The lifecycle verbs take sandboxd's operator path under the fleet root token: -// a per-sandbox secret would turn the control plane's O(nodes) storage into -// O(sandboxes). - // ForkSpec is the POST /v1/sandboxes/{id}/fork body. Token stays empty on the // operator path; Count must be within the node's max_fork_count. type ForkSpec struct { @@ -57,23 +53,6 @@ type PoolKey struct { Engine string `json:"engine,omitempty"` } -// CheckpointClaimSpec is the POST /v1/checkpoints/{id}/claim body. -type CheckpointClaimSpec struct { - TTLSeconds int `json:"ttl_seconds,omitempty"` -} - -// PromoteSpec is the POST /v1/sandboxes/{id}/promote body: it publishes the -// sandbox's state as a node-local template future claims clone from. -type PromoteSpec struct { - Token string `json:"token,omitempty"` - Template string `json:"template"` -} - -// PromoteResult returns the promoted template's full key. -type PromoteResult struct { - Key PoolKey `json:"key"` -} - // SandboxSummary is one live claim as the owning node reports it. type SandboxSummary struct { ID string `json:"id"` @@ -138,17 +117,6 @@ func (c *Client) Checkpoint(ctx context.Context, id string, spec CheckpointSpec) return out.Checkpoint, err } -// ClaimCheckpoint performs POST /v1/checkpoints/{id}/claim, delivering a fresh -// sandbox branched from the checkpoint's exact state. -func (c *Client) ClaimCheckpoint(ctx context.Context, checkpointID string, spec CheckpointClaimSpec) (ClaimResult, error) { - var out ClaimResult - if checkpointID == "" { - return out, fmt.Errorf("sandboxd: claim checkpoint requires a checkpoint id") - } - err := c.postJSON(ctx, "/v1/checkpoints/"+url.PathEscape(checkpointID)+"/claim", spec, &out) - return out, err -} - // Checkpoints performs GET /v1/checkpoints, newest first. func (c *Client) Checkpoints(ctx context.Context) ([]Checkpoint, error) { var out struct { @@ -163,36 +131,7 @@ func (c *Client) DeleteCheckpoint(ctx context.Context, checkpointID string) erro if checkpointID == "" { return fmt.Errorf("sandboxd: delete checkpoint requires a checkpoint id") } - u := c.baseURL + "/v1/checkpoints/" + url.PathEscape(checkpointID) - req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u, nil) - if err != nil { - return err - } - c.authenticate(req, c.token) - - resp, err := c.hc.Do(req) - if err != nil { - return fmt.Errorf("sandboxd: delete checkpoint: %w", err) - } - defer drainAndClose(resp) - - switch resp.StatusCode { - case http.StatusNoContent, http.StatusNotFound: - return nil - default: - return statusError(resp) - } -} - -// Promote performs POST /v1/sandboxes/{id}/promote, publishing the sandbox as -// a node-local template that later claims for that key clone from. -func (c *Client) Promote(ctx context.Context, id string, spec PromoteSpec) (PoolKey, error) { - if id == "" { - return PoolKey{}, fmt.Errorf("sandboxd: promote requires a sandbox id") - } - var out PromoteResult - err := c.postJSON(ctx, "/v1/sandboxes/"+url.PathEscape(id)+"/promote", spec, &out) - return out.Key, err + return c.sendNoBody(ctx, http.MethodDelete, "/v1/checkpoints/"+url.PathEscape(checkpointID), c.token, "delete checkpoint", http.StatusNoContent, http.StatusNotFound) } // Stats performs GET /v1/sandboxes/{id}/stats. @@ -219,23 +158,7 @@ func (c *Client) sandboxVerb(ctx context.Context, id, verb string) error { if id == "" { return fmt.Errorf("sandboxd: %s requires a sandbox id", verb) } - u := c.baseURL + "/v1/sandboxes/" + url.PathEscape(id) + "/" + verb - req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil) - if err != nil { - return err - } - c.authenticate(req, c.token) - - resp, err := c.hc.Do(req) - if err != nil { - return fmt.Errorf("sandboxd: %s: %w", verb, err) - } - defer drainAndClose(resp) - - if resp.StatusCode != http.StatusNoContent { - return statusError(resp) - } - return nil + return c.sendNoBody(ctx, http.MethodPost, "/v1/sandboxes/"+url.PathEscape(id)+"/"+verb, c.token, verb, http.StatusNoContent) } // postJSON sends body as JSON via POST and decodes a 2xx reply into out. diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index bba6dca..2656744 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -11,6 +11,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "os" @@ -35,6 +36,7 @@ import ( sandboxv1beta1 "github.com/cocoonstack/sandbox-operator/api/v1beta1" extv1alpha1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1alpha1" extv1beta1 "github.com/cocoonstack/sandbox-operator/extensions/api/v1beta1" + "github.com/cocoonstack/sandbox-operator/test/benchutil" ) const image = "m.daocloud.io/docker.io/library/alpine:3.20" @@ -62,19 +64,19 @@ type result struct { func main() { flag.Parse() rootCtx := context.Background() - must(clientgoscheme.AddToScheme(scheme)) - must(sandboxv1beta1.AddToScheme(scheme)) - must(sandboxv1alpha1.AddToScheme(scheme)) - must(extv1beta1.AddToScheme(scheme)) - must(extv1alpha1.AddToScheme(scheme)) + benchutil.Must(clientgoscheme.AddToScheme(scheme)) + benchutil.Must(sandboxv1beta1.AddToScheme(scheme)) + benchutil.Must(sandboxv1alpha1.AddToScheme(scheme)) + benchutil.Must(extv1beta1.AddToScheme(scheme)) + benchutil.Must(extv1alpha1.AddToScheme(scheme)) var err error cfg, err = clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) - must(err) + benchutil.Must(err) cl, err = ctrlclient.New(cfg, ctrlclient.Options{Scheme: scheme}) - must(err) + benchutil.Must(err) cs, err = kubernetes.NewForConfig(cfg) - must(err) + benchutil.Must(err) ensureNS(rootCtx, *ns) @@ -133,13 +135,6 @@ func main() { } } -func must(err error) { - if err != nil { - fmt.Fprintln(os.Stderr, "fatal:", err) - os.Exit(2) - } -} - func ensureNS(ctx context.Context, n string) { nsObj := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: n, Labels: map[string]string{"cocoon-e2e-run": *run}}} _ = cl.Create(ctx, nsObj) @@ -374,6 +369,22 @@ func scSuspendResume(ctx context.Context) (string, error) { return "suspend removed pod; resume recreated + Ready", nil } +// pollDetail retries probe on a 3s tick until it yields evidence or d elapses; an empty string means keep polling. +func pollDetail(d time.Duration, timeoutMsg string, probe func() (string, error)) (string, error) { + deadline := time.Now().Add(d) + for time.Now().Before(deadline) { + detail, err := probe() + if err != nil { + return "", err + } + if detail != "" { + return detail, nil + } + time.Sleep(3 * time.Second) + } + return "", errors.New(timeoutMsg) +} + func waitPodGone(ctx context.Context, name string, d time.Duration) error { deadline := time.Now().Add(d) for time.Now().Before(deadline) { @@ -400,8 +411,7 @@ func scShutdownRetain(ctx context.Context) (string, error) { } defer deleteSandbox(ctx, name) // With Retain + past shutdownTime, the sandbox should end up Expired (not deleted). - deadline := time.Now().Add(90 * time.Second) - for time.Now().Before(deadline) { + return pollDetail(90*time.Second, "no Expired condition within deadline", func() (string, error) { s := &sandboxv1beta1.Sandbox{} if err := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, s); err != nil { return "", err @@ -414,9 +424,8 @@ func scShutdownRetain(ctx context.Context) (string, error) { return "expiry reason observed: " + c.Reason, nil } } - time.Sleep(3 * time.Second) - } - return "", fmt.Errorf("no Expired condition within deadline") + return "", nil + }) } func scPVC(ctx context.Context) (string, error) { @@ -437,8 +446,7 @@ func scPVC(ctx context.Context) (string, error) { return "", err } defer deleteSandbox(ctx, name) - deadline := time.Now().Add(120 * time.Second) - for time.Now().Before(deadline) { + return pollDetail(120*time.Second, "no PVC created for sandbox", func() (string, error) { pvcs := &corev1.PersistentVolumeClaimList{} if err := cl.List(ctx, pvcs, ctrlclient.InNamespace(*ns)); err == nil { for _, p := range pvcs.Items { @@ -447,16 +455,14 @@ func scPVC(ctx context.Context) (string, error) { } } } - time.Sleep(3 * time.Second) - } - return "", fmt.Errorf("no PVC created for sandbox") + return "", nil + }) } func scDeleteCleanup(ctx context.Context) (string, error) { name := "e2e-core" deleteSandbox(ctx, name) - deadline := time.Now().Add(90 * time.Second) - for time.Now().Before(deadline) { + return pollDetail(90*time.Second, "resources not fully cleaned up", func() (string, error) { s := &sandboxv1beta1.Sandbox{} errS := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, s) p := &corev1.Pod{} @@ -466,9 +472,8 @@ func scDeleteCleanup(ctx context.Context) (string, error) { if apierrors.IsNotFound(errS) && apierrors.IsNotFound(errP) && apierrors.IsNotFound(errSvc) { return "sandbox+pod+service fully garbage-collected", nil } - time.Sleep(3 * time.Second) - } - return "", fmt.Errorf("resources not fully cleaned up") + return "", nil + }) } func newTemplate(name string) *extv1beta1.SandboxTemplate { @@ -522,8 +527,7 @@ func scWarmPoolScale(ctx context.Context) (string, error) { if err := cl.Create(ctx, wp); err != nil { return "", err } - deadline := time.Now().Add(150 * time.Second) - for time.Now().Before(deadline) { + return pollDetail(150*time.Second, "warm pool did not reach desired replicas", func() (string, error) { sl := &sandboxv1beta1.SandboxList{} if err := cl.List(ctx, sl, ctrlclient.InNamespace(*ns)); err == nil { warm := 0 @@ -538,9 +542,8 @@ func scWarmPoolScale(ctx context.Context) (string, error) { return fmt.Sprintf("warm pool provisioned %d sandboxes", warm), nil } } - time.Sleep(3 * time.Second) - } - return "", fmt.Errorf("warm pool did not reach desired replicas") + return "", nil + }) } func scClaimWarmHit(ctx context.Context) (string, error) { @@ -557,8 +560,7 @@ func scClaimWarmHit(ctx context.Context) (string, error) { return "", err } defer deleteClaim(ctx, name) - deadline := time.Now().Add(120 * time.Second) - for time.Now().Before(deadline) { + return pollDetail(120*time.Second, "claim did not bind to a warm sandbox", func() (string, error) { got := &extv1beta1.SandboxClaim{} if err := cl.Get(ctx, types.NamespacedName{Namespace: *ns, Name: name}, got); err == nil { if got.Status.SandboxStatus.Name != "" { @@ -570,9 +572,8 @@ func scClaimWarmHit(ctx context.Context) (string, error) { } } } - time.Sleep(3 * time.Second) - } - return "", fmt.Errorf("claim did not bind to a warm sandbox") + return "", nil + }) } func scConversion(ctx context.Context) (string, error) { diff --git a/test/e2ebench/main.go b/test/e2ebench/main.go index 9851b06..fd87bbc 100644 --- a/test/e2ebench/main.go +++ b/test/e2ebench/main.go @@ -74,18 +74,16 @@ const ( runVal = "g0131-phase-d" ) -func must(err error) { benchutil.Must(err) } - func main() { flag.Parse() - must(clientgoscheme.AddToScheme(scheme)) - must(sandboxv1beta1.AddToScheme(scheme)) - must(extv1beta1.AddToScheme(scheme)) + benchutil.Must(clientgoscheme.AddToScheme(scheme)) + benchutil.Must(sandboxv1beta1.AddToScheme(scheme)) + benchutil.Must(extv1beta1.AddToScheme(scheme)) cfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) - must(err) + benchutil.Must(err) cfg.QPS, cfg.Burst = 200, 400 cl, err = ctrlclient.New(cfg, ctrlclient.Options{Scheme: scheme}) - must(err) + benchutil.Must(err) ctx := context.Background() res := map[string]any{ @@ -140,7 +138,7 @@ func main() { res["pass"] = pass b, _ := json.MarshalIndent(res, "", " ") - must(os.WriteFile(*out, b, 0o644)) + benchutil.Must(os.WriteFile(*out, b, 0o644)) fmt.Printf("pass=%v; wrote %s\n", pass, *out) if !pass { os.Exit(1) @@ -182,7 +180,7 @@ func ensureTemplate(ctx context.Context) { }, } if err := cl.Create(ctx, t); err != nil && !apierrors.IsAlreadyExists(err) { - must(err) + benchutil.Must(err) } } diff --git a/test/l2bench/main.go b/test/l2bench/main.go index 693a1b4..e189288 100644 --- a/test/l2bench/main.go +++ b/test/l2bench/main.go @@ -57,8 +57,6 @@ var ( orphansFlag = flag.Int("orphans", 200, "orphan bindings to inject and reconcile") ) -func must(err error) { benchutil.Must(err) } - // fakeSandboxd is an httptest-backed sandboxd: /v1/claim hands over a fresh // sandbox fast; /v1/sandboxes/{id}/release counts VM destroys (must stay 0). type fakeSandboxd struct { @@ -119,8 +117,8 @@ func (s sliceInventory) LiveDeliveries(context.Context) ([]scale.Delivery, error func newScheme() *runtime.Scheme { s := runtime.NewScheme() - must(sandboxv1beta1.AddToScheme(s)) - must(extv1beta1.AddToScheme(s)) + benchutil.Must(sandboxv1beta1.AddToScheme(s)) + benchutil.Must(extv1beta1.AddToScheme(s)) return s } @@ -149,9 +147,9 @@ func measureClaimLatency(ctx context.Context, gw claimWaiter, iters int) []float start := time.Now() a, err := gw.Claim(ctx, req) d := time.Since(start) - must(err) + benchutil.Must(err) if a.SandboxName == "" { - must(fmt.Errorf("empty assignment at iter %d", i)) + benchutil.Must(fmt.Errorf("empty assignment at iter %d", i)) } lat = append(lat, float64(d.Nanoseconds())/1e6) } @@ -182,7 +180,7 @@ func injectAndReconcileOrphans(ctx context.Context, fs *fakeSandboxd, orphans in for i := range orphans { name := fmt.Sprintf("orphan-%d", i) a, err := badGW.Claim(ctx, scale.ClaimRequest{Namespace: ns, ClaimName: name, WarmPool: "base:24.04"}) - must(err) + benchutil.Must(err) inv = append(inv, scale.Delivery{ SandboxName: a.SandboxName, Node: a.Node, Address: a.Address, ClaimNS: ns, ClaimName: name, }) @@ -191,12 +189,12 @@ func injectAndReconcileOrphans(ctx context.Context, fs *fakeSandboxd, orphans in orc := scale.NewOrphanReconciler(node, inv, fc, scale.NewClaimRecorder(fc), logr.Discard()) reconciled, err := orc.Reconcile(ctx) - must(err) + benchutil.Must(err) remaining := 0 for i := range orphans { cur := &extv1beta1.SandboxClaim{} - must(fc.Get(ctx, types.NamespacedName{Namespace: ns, Name: fmt.Sprintf("orphan-%d", i)}, cur)) + benchutil.Must(fc.Get(ctx, types.NamespacedName{Namespace: ns, Name: fmt.Sprintf("orphan-%d", i)}, cur)) if cur.Status.SandboxStatus.Name == "" { remaining++ } @@ -241,9 +239,9 @@ func main() { "claim_iters": *itersFlag, } b, err := json.MarshalIndent(out, "", " ") - must(err) - must(os.MkdirAll(filepath.Dir(*outFlag), 0o755)) - must(os.WriteFile(*outFlag, b, 0o644)) + benchutil.Must(err) + benchutil.Must(os.MkdirAll(filepath.Dir(*outFlag), 0o755)) + benchutil.Must(os.WriteFile(*outFlag, b, 0o644)) fmt.Printf("claim p50=%.4fms p95=%.4fms (iters=%d) | orphans injected=%d reconciled=%d remaining=%d | vm_destroy_calls=%d | pass=%v\n", p50, p95, *itersFlag, *orphansFlag, reconciled, remaining, destroys, pass) diff --git a/test/l3bench/main.go b/test/l3bench/main.go index 71836af..915499b 100644 --- a/test/l3bench/main.go +++ b/test/l3bench/main.go @@ -58,8 +58,6 @@ var ( namespacesFlag = flag.Int("namespaces", 3, "number of namespaces to spread sandboxes across") ) -func must(err error) { benchutil.Must(err) } - func fail(format string, args ...any) { fmt.Fprintf(os.Stderr, "FAIL: "+format+"\n", args...) os.Exit(1) @@ -119,7 +117,7 @@ func main() { Address: fmt.Sprintf("10.%d.%d.%d:7777", k, (i>>8)&0xff, i&0xff), }) } - must(source.Apply(ctx, &scale.NodeInventory{ + benchutil.Must(source.Apply(ctx, &scale.NodeInventory{ Kind: scale.NodeInventoryGVK.Kind, APIVersion: scale.NodeInventoryGVK.GroupVersion().String(), Name: node, @@ -145,7 +143,7 @@ func main() { store := scale.NewScatterGatherStore(source, scale.WithLogger(logr.Discard()), scale.WithWatchPollInterval(50*time.Millisecond)) server, err := sandboxapiserver.NewInProcessServer("l3bench-apiserver", store) - must(err) + benchutil.Must(err) ts := httptest.NewServer(server.Handler) defer ts.Close() @@ -164,7 +162,7 @@ func main() { fail("client-go namespaced list failed: %v", err) } wantNS, err := store.List(ctx, scale.ListOptions{Namespace: sampleNS}) - must(err) + benchutil.Must(err) if len(nsList.Items) != len(wantNS.Items) || len(nsList.Items) == 0 { fail("namespaced list returned %d, want %d (>0)", len(nsList.Items), len(wantNS.Items)) } @@ -213,9 +211,9 @@ func main() { "substrate": "in-process genericapiserver via httptest + client-go", } b, err := json.MarshalIndent(out, "", " ") - must(err) - must(os.MkdirAll(filepath.Dir(*outFlag), 0o755)) - must(os.WriteFile(*outFlag, b, 0o644)) + benchutil.Must(err) + benchutil.Must(os.MkdirAll(filepath.Dir(*outFlag), 0o755)) + benchutil.Must(os.WriteFile(*outFlag, b, 0o644)) fmt.Printf("sandboxes served=%d | etcd objects=%d (nodes=%d + pools=%d) | ssa writes=%d | per-sandbox etcd objects=0\n", len(allList.Items), etcdObjectCount, nodes, pools, source.ApplyCount()) @@ -237,7 +235,7 @@ func newRESTClient(host string) *restclient.RESTClient { cfg.ContentConfig.NegotiatedSerializer = sandboxapiserver.Codecs.WithoutConversion() cfg.ContentConfig.ContentType = "application/json" rc, err := restclient.RESTClientFor(cfg) - must(err) + benchutil.Must(err) return rc } diff --git a/test/poolbench/main.go b/test/poolbench/main.go index 89928e7..6684678 100644 --- a/test/poolbench/main.go +++ b/test/poolbench/main.go @@ -62,22 +62,20 @@ const ( poolName = "poolbench-pool" ) -func must(err error) { benchutil.Must(err) } - func main() { flag.Parse() ctx := context.Background() - must(clientgoscheme.AddToScheme(scheme)) - must(sandboxv1beta1.AddToScheme(scheme)) - must(extv1beta1.AddToScheme(scheme)) + benchutil.Must(clientgoscheme.AddToScheme(scheme)) + benchutil.Must(sandboxv1beta1.AddToScheme(scheme)) + benchutil.Must(extv1beta1.AddToScheme(scheme)) var err error cfg, err = clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) - must(err) + benchutil.Must(err) cfg.QPS, cfg.Burst = 400, 800 cl, err = ctrlclient.New(cfg, ctrlclient.Options{Scheme: scheme}) - must(err) + benchutil.Must(err) wcl, err = ctrlclient.NewWithWatch(cfg, ctrlclient.Options{Scheme: scheme}) - must(err) + benchutil.Must(err) result := map[string]any{"ns": *ns, "poolTarget": *poolSize} @@ -168,7 +166,7 @@ func ensureTemplate(ctx context.Context) { } err := cl.Create(ctx, t) if err != nil && !apierrors.IsAlreadyExists(err) { - must(err) + benchutil.Must(err) } } diff --git a/test/scalebench/main.go b/test/scalebench/main.go index 7189833..743a905 100644 --- a/test/scalebench/main.go +++ b/test/scalebench/main.go @@ -68,13 +68,11 @@ var ( maxPasses = flag.Int("max-passes", 8, "max reconcile passes per claim before giving up") ) -func must(err error) { benchutil.Must(err) } - func newScheme() *runtime.Scheme { s := runtime.NewScheme() - must(sandboxv1beta1.AddToScheme(s)) - must(extv1beta1.AddToScheme(s)) - must(corev1.AddToScheme(s)) + benchutil.Must(sandboxv1beta1.AddToScheme(s)) + benchutil.Must(extv1beta1.AddToScheme(s)) + benchutil.Must(corev1.AddToScheme(s)) return s } @@ -273,12 +271,12 @@ func main() { var sizes []int for _, s := range strings.Split(*sizesFlag, ",") { v, err := strconv.Atoi(strings.TrimSpace(s)) - must(err) + benchutil.Must(err) sizes = append(sizes, v) } sort.Ints(sizes) if len(sizes) < 2 { - must(fmt.Errorf("need at least two sizes to measure a ratio, got %v", sizes)) + benchutil.Must(fmt.Errorf("need at least two sizes to measure a ratio, got %v", sizes)) } results := make([]sizeResult, 0, len(sizes)) @@ -327,7 +325,7 @@ func main() { "results": results, } b, _ := json.MarshalIndent(out, "", " ") - must(os.WriteFile(*outFlag, b, 0o644)) + benchutil.Must(os.WriteFile(*outFlag, b, 0o644)) fmt.Printf("fast p50 ratio (N=%d/N=%d) = %.3f (threshold %.1f); naive ratio = %.3f; pass=%v\n", hi.N, lo.N, fastRatio, *threshold, naiveRatio, pass) fmt.Printf("wrote %s\n", *outFlag) diff --git a/test/scalestress/main.go b/test/scalestress/main.go index bbf07b2..1ce3aff 100644 --- a/test/scalestress/main.go +++ b/test/scalestress/main.go @@ -69,26 +69,24 @@ const ( runVal = "g0131-stress" ) -func must(err error) { benchutil.Must(err) } - func main() { flag.Parse() - must(clientgoscheme.AddToScheme(scheme)) - must(sandboxv1beta1.AddToScheme(scheme)) - must(extv1beta1.AddToScheme(scheme)) + benchutil.Must(clientgoscheme.AddToScheme(scheme)) + benchutil.Must(sandboxv1beta1.AddToScheme(scheme)) + benchutil.Must(extv1beta1.AddToScheme(scheme)) cfg, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) - must(err) + benchutil.Must(err) cfg.QPS, cfg.Burst = 200, 400 cl, err = ctrlclient.New(cfg, ctrlclient.Options{Scheme: scheme}) - must(err) + benchutil.Must(err) cs, err = kubernetes.NewForConfig(cfg) - must(err) + benchutil.Must(err) hosts := splitCSV(*hostsCSV) steps := parseSteps(*stepsCSV) for _, n := range steps { if n > 200 { - must(fmt.Errorf("refusing step %d: this run is capped at 200 sandboxes", n)) + benchutil.Must(fmt.Errorf("refusing step %d: this run is capped at 200 sandboxes", n)) } } ctx := context.Background() @@ -96,7 +94,7 @@ func main() { prodBase := prodPods(ctx, *prodNS, hosts) fmt.Printf("[prod] baseline on %v = %d pods (hard guard)\n", hosts, prodBase) if prodBase <= 0 { - must(fmt.Errorf("refusing to run: prod baseline non-positive (%d)", prodBase)) + benchutil.Must(fmt.Errorf("refusing to run: prod baseline non-positive (%d)", prodBase)) } ensureNS(ctx) @@ -219,7 +217,7 @@ func main() { result["completed_clean"] = abort == "" && len(rounds) == len(steps) && prodOK b, _ := json.MarshalIndent(result, "", " ") - must(os.WriteFile(*out, b, 0o644)) + benchutil.Must(os.WriteFile(*out, b, 0o644)) fmt.Printf("\n== new LIST time-out rejections during ramp: %.0f | peak %s seats in use: %.0f/%.0f | client LIST-sb p95 x%.2f | wedge_observed=%v | prod intact=%v ==\n", newRejections, *plName, maxInUse, nominal, ratio, wedgeObserved, prodOK) fmt.Printf("wrote %s\n", *out) @@ -274,7 +272,7 @@ func ensureTemplate(ctx context.Context, hosts []string) { }, } if err := cl.Create(ctx, t); err != nil && !apierrors.IsAlreadyExists(err) { - must(err) + benchutil.Must(err) } } @@ -455,7 +453,7 @@ func parseSteps(s string) []int { var out []int for _, p := range splitCSV(s) { v, err := strconv.Atoi(p) - must(err) + benchutil.Must(err) out = append(out, v) } return out