diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml new file mode 100644 index 000000000..ab99627f8 --- /dev/null +++ b/.github/workflows/e2e-ipv6.yaml @@ -0,0 +1,504 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e-ipv6 +# Separate from pr-workflow.yaml so this can be gated independently -- and so an +# hour-long IPv6 run never delays that workflow's merge-gating jobs. +# +# Only the networking suite runs. Every test that can tell one address family +# from another lives in it, and the demo suite is a large share of the wall +# clock for coverage that is not family-specific. The demos still deploy, +# because networking builds its actors from their templates. +# +# Both sandbox classes run, gVisor then micro-VM, off the one cluster and the +# one E2E_SANDBOX_CLASS knob. The class is where address-family support +# diverges: gVisor's runsc adopts the interior netns wholesale and picks the +# actor's IPv6 up for free, while a micro-VM's eth0 is a tap cross-connected to +# that netns at L2, so the guest has to be told over the kata-agent channel. A +# lane that ran gVisor alone could not tell a working micro-VM guest from one +# that never got an address. +# +# Two ways in. A pull request that edits this file runs it: the paths filter is +# evaluated before the job, so no unrelated request pays for a v6 cluster, and +# the lane can demonstrate itself without a label someone has to create first. +# Dispatch it with a list of pull requests and it merges them onto main in that +# order and runs the result: the IPv6 work is a set of changes that do nothing +# apart, so a per-request run cannot show the feature working, only that it did +# no harm. The stack exists for the length of the run and is never pushed, +# which is what lets this branch carry the workflow file alone instead of a +# copy of every change it wants to exercise. +# +# Stacking is dispatch-only, so the pull_request path runs main plus this file +# and nothing else. That exercises the lane, not the IPv6 work, and it stays +# red until the changes this gates are on main. +# +# Either way this job stays out of the `e2e-test` gate, so it never blocks a PR. +on: + pull_request: + paths: ['.github/workflows/e2e-ipv6.yaml'] + workflow_dispatch: + inputs: + prs: + description: 'Pull requests to merge onto main, in order, e.g. "958 938 1083". Blank runs main alone.' + required: false + type: string +permissions: + contents: read +jobs: + e2e-test-ipv6: + runs-on: ubuntu-latest + # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's + # 6-hour default. A broken IPv6 cluster does not crash, it misses + # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. + # Raised from 40 for the second sandbox class: a micro-VM golden snapshot is + # a cloud-hypervisor cold boot plus a checkpoint on nested KVM, and the + # networking suite then runs twice. + timeout-minutes: 75 + env: + # Non-default name so these steps can be replayed locally without + # touching an existing cluster. install-ate-kind.sh does not derive + # KUBECTL_CONTEXT from the cluster name the way run-e2e-kind.sh does, + # so both have to be set here. + KIND_CLUSTER_NAME: ate-ipv6 + KUBECTL_CONTEXT: kind-ate-ipv6 + # 8.8.8.8 reached through the well-known NAT64 prefix. CoreDNS is a + # v6-only pod on a runner with no IPv6 egress of its own, so this is the + # only shape of upstream resolver it can reach. Read back below, not + # passed in: the script derives it from the prefix, and a job that + # supplied it would not notice if that stopped working. + EXPECT_DNS_UPSTREAM: 64:ff9b::0808:0808 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # The merges below need refs this clone would not otherwise have. + fetch-depth: 0 + - name: Stack the pull requests under test + # Merge, not cherry-pick: a request that has been squash-merged since it + # was opened re-applies as a revert of what is already on main, and only + # a merge notices there is nothing to do. Order is the order given, so a + # conflict names the request that has to rebase rather than the stack. + # + # This builds and runs the requests' code. The token is read-only and no + # secrets reach it; a run happens only when someone asks for one. + if: github.event_name == 'workflow_dispatch' && inputs.prs != '' + env: + GH_TOKEN: ${{ github.token }} + PRS: ${{ inputs.prs }} + # The numbers in `prs` are always upstream pull requests, so they are + # resolved against upstream by name rather than against whichever repo + # the run happens to be in. Reading them needs no token. + UPSTREAM: https://github.com/agent-substrate/substrate + run: | + git config user.name 'e2e-ipv6' + git config user.email 'e2e-ipv6@invalid' + { + echo "### Stack under test" + echo "" + echo "Merged onto \`main\` at \`$(git rev-parse --short HEAD)\`, in order:" + echo "" + echo "| pull request | head | title |" + echo "|---|---|---|" + } >> "$GITHUB_STEP_SUMMARY" + for pr in ${PRS//,/ }; do + case "${pr}" in + ''|*[!0-9]*) echo "::error::'${pr}' is not a pull request number"; exit 1 ;; + esac + git fetch -q "${UPSTREAM}" "refs/pull/${pr}/head:pr-${pr}" \ + || { echo "::error::no such pull request: #${pr}"; exit 1; } + if ! git merge --no-edit -q "pr-${pr}"; then + echo "::error::#${pr} does not merge onto the stack; it needs a rebase" + git merge --abort || true + exit 1 + fi + title=$(gh pr view "${pr}" --repo "${UPSTREAM}" --json title -q .title 2>/dev/null || echo '?') + echo "| #${pr} | \`$(git rev-parse --short "pr-${pr}")\` | ${title} |" >> "$GITHUB_STEP_SUMMARY" + echo "merged #${pr}: ${title}" + done + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: 'go.mod' + - name: Free disk space + # kind node image + control-plane images + snapshots are tight on the + # ~14GB runner disk, and the micro-VM assets -- a 1GB kata-static tarball + # unpacked to a 256MB rootfs, plus a kernel -- are on top of that. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Assert the micro-VM IPv6 staging fix is in this tree + # Named check for the same reason as the nat64 one below. On a v6-only + # cluster the rustfs ClusterIP is a bare IPv6 literal, which is not a URL, + # so asset staging fails outright without this -- ten minutes in, as an + # aws-cli error about an endpoint, naming nothing that would lead here. + run: | + grep -q 'RUSTFS_IP="\[' hack/microvm-assets/stage-to-rustfs.sh || { + echo "::error::hack/microvm-assets/stage-to-rustfs.sh does not bracket an IPv6 rustfs ClusterIP -- stack #1122 to run the micro-VM lane" + exit 1 + } + - name: Cache micro-VM assets + # restore, not the composite cache action: pr-workflow.yaml's e2e-test + # writes this key on every push to main, and a second saver would race it + # for no gain -- the assets are identical and assemble.sh fully pins them. + # A miss just means run-microvm-demo-kind.sh assembles them itself. + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable IPv6 in the Docker daemon + # ubuntu-latest ships dockerd with IPv6 off, so kind would create its + # network v4-only and create-kind-cluster.sh would reject the cluster. + # Merge the two keys into whatever daemon.json the runner image ships + # rather than replacing the file. + run: | + sudo mkdir -p /etc/docker + [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo cat /etc/docker/daemon.json \ + | jq '. + {"ipv6": true, "ip6tables": true}' \ + | sudo tee /etc/docker/daemon.json.new >/dev/null + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + sudo systemctl restart docker + # Assert, don't print: if the merge or the restart silently did not take, + # kind builds a v4-only network and the failure surfaces much later as a + # cluster that is not the one this job exists to test. + docker network inspect bridge --format '{{.EnableIPv6}}' | grep -qx true || { + echo "::error::dockerd did not come back with IPv6 enabled" + sudo cat /etc/docker/daemon.json; exit 1; + } + echo "bridge EnableIPv6=true" + - name: Enable forwarding for the NAT64 agent + # ubuntu-latest has no IPv6 egress whatsoever -- measured, not assumed: + # every curl -6 fails in ~2ms. A v6-only cluster still has to reach real + # v4 destinations (atelet fetches the gVisor tarball from GCS, + # TestActorEgress fetches example.com), so the cluster translates for + # itself. What upstream's own e2e sets, and ordered after the dockerd + # restart, which rebuilds the chains this affects. + run: | + sudo sysctl -w net.ipv4.ip_forward=1 + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + - name: Enable KVM + # Grant the runner access to /dev/kvm so create-kind-cluster.sh makes it + # usable inside the node. No node label follows from this: a micro-VM + # WorkerPool reaches the node through the ate.dev/kvm extended resource + # atelet's device plugin advertises, which is why the check that this + # worked has to wait until after the install. After the dockerd restart + # above, which is what the script's probe container runs under. + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + # NAT64 is opt-in in the script and this is the case it exists for: the + # names this job needs have real AAAA records pointing at addresses the + # runner cannot reach, so the prefix deploys the translator and routes + # cluster DNS through it. The script gates on a pod fetching through the + # prefix, so a broken translator fails here rather than as a rollout + # timeout ten minutes later. Everything the Corefile ends up containing + # is the script's doing -- the steps below only read it back. + env: + IP_FAMILY: ipv6 + IPV6_DNS64_PREFIX: '64:ff9b::/96' + run: | + # Named check rather than letting the script report a missing file: + # this is the first step that touches the IPv6-only kind support, so on + # a tree without it this is where the run stops, and it should say why. + [ -f hack/third_party/nat64/install.yaml ] || { + echo "::error::hack/third_party/nat64/install.yaml is missing -- the IPv6-only kind support this job gates is not in this tree" + exit 1 + } + hack/create-kind-cluster.sh + - name: Wait for the node to be ready + # podCIDRs is assigned by the controller-manager after the node registers, + # so the assertion below can read an empty field on a cluster that is + # perfectly fine and report "cannot confirm IP family". Poll for the field + # this job actually reads, then wait on Ready for everything after it. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + for _ in $(seq 60); do + [ -n "$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}' 2>/dev/null)" ] && break + sleep 2 + done + k wait --for=condition=Ready nodes --all --timeout=180s + - name: Assert the node has /dev/kvm + # create-kind-cluster.sh probes for KVM and degrades silently to + # gVisor-only when the probe fails, so without this a "micro-VM lane" + # could pass by never having been one. Only the device is checkable here; + # what the scheduler actually places on is asserted after the install, + # because atelet is what advertises it. + run: | + for node in $(hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + docker exec "${node}" test -c /dev/kvm \ + || { echo "::error::${node} has no /dev/kvm -- the micro-VM lane would be gVisor in disguise"; exit 1; } + echo " ${node}: /dev/kvm present" + done + - name: Assert the cluster is single-stack IPv6 + # This job is worthless if the cluster is not actually v6-only, and a + # green run leaves no evidence either way -- the diagnostics dump below + # only runs on failure. A kind default change or an IP_FAMILY regression + # would otherwise turn this into a second IPv4 run that reports success. + # Checked here rather than later so it fails as itself. + # + # PreferDualStack Services resolving to a single clusterIP is the + # positive signal: on a dual-stack cluster they would get two. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + pod_cidrs=$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}') + svc_ips=$(k -n default get svc kubernetes -o jsonpath='{.spec.clusterIPs[*]}') + node_ips=$(k get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}') + for pair in "podCIDRs=${pod_cidrs}" "kubernetes.clusterIPs=${svc_ips}" "node.InternalIP=${node_ips}"; do + case "${pair#*=}" in + *.*) echo "::error::not single-stack IPv6 -- ${pair}"; exit 1 ;; + "") echo "::error::empty, cannot confirm IP family -- ${pair}"; exit 1 ;; + esac + echo " ${pair}" + done + echo "single-stack IPv6 confirmed" + - name: Assert create-kind-cluster.sh built the Corefile this job needs + # The script owns the whole Corefile, so this reads it back instead of + # rewriting it. That is the point: a green run is then evidence for what + # the script produced, where a job that patched the Corefile itself would + # paper over the absence of the very change it is here to gate. Each of + # these is something the install would otherwise fail on ten minutes + # later and far less legibly. + run: | + Corefile=$(kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}') + echo "${Corefile}" + check() { + echo "${Corefile}" | grep -q "$1" || { echo "::error::$2"; exit 1; } + } + if echo "${Corefile}" | grep -q 'forward .* /etc/resolv.conf'; then + echo "::error::CoreDNS still forwards to /etc/resolv.conf -- the IPv6-only DNS fix is not in this tree" + exit 1 + fi + check "forward .* ${EXPECT_DNS_UPSTREAM}" \ + "CoreDNS does not forward to ${EXPECT_DNS_UPSTREAM}" + check '^kind-registry:53' \ + "no kind-registry server block -- atelet cannot pull from its own netns" + check '^cluster.local:53' \ + "the cluster zones have no server block of their own -- dns64 would take out service discovery" + check 'translate_all' \ + "no dns64 translate_all -- external names resolve to addresses this runner cannot reach" + echo "Corefile is the IPv6-only shape this job needs" + - name: Verify cluster DNS answers both internal and external names + # The install is the next step and it takes ten minutes to fail. A DNS + # regression is the failure this Corefile is most likely to cause, so + # assert all three cases here where the message is unambiguous. + # Reads the log rather than attaching. `kubectl run --attach` has to + # connect before the container writes, and on a cluster still settling it + # loses that race far more often than it wins it -- a probe that fails on + # a healthy cluster is worse than no probe. Create, wait, read the log, + # and retry the whole thing. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + probe() { + k delete pod dnscheck --ignore-not-found --wait --timeout=60s >/dev/null 2>&1 + k run dnscheck --restart=Never --image=busybox:1.36 --command -- \ + sh -c ' + nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 \ + || { echo "FAIL: an in-cluster Service does not resolve"; exit 1; } + nslookup storage.googleapis.com 2>/dev/null | grep -q "64:ff9b" \ + || { echo "FAIL: external names are not synthesized through NAT64"; exit 1; } + # Absolute name deliberately. The registry has its own server + # block and no search domain leads to it, so the bare name is + # NXDOMAIN under ndots:5 -- busybox gives up there, while glibc + # and Go go on to try the name as given, which is what atelet + # does when it pulls from its own netns. Querying what those + # resolvers end up querying is what makes this worth failing on. + nslookup kind-registry. >/dev/null 2>&1 \ + || { echo "FAIL: kind-registry does not resolve; atelet cannot pull"; exit 1; } + echo "DNS-OK" + ' >/dev/null 2>&1 || return 1 + k wait --for=jsonpath='{.status.phase}'=Succeeded pod/dnscheck --timeout=120s \ + >/dev/null 2>&1 + k logs dnscheck > /tmp/dnscheck.log 2>&1 || return 1 + grep -q DNS-OK /tmp/dnscheck.log + } + for attempt in $(seq 6); do + if probe; then cat /tmp/dnscheck.log; exit 0; fi + echo "dnscheck attempt ${attempt} failed" + cat /tmp/dnscheck.log 2>/dev/null || true + sleep 10 + done + echo "::error::cluster DNS did not come up" + exit 1 + - name: Install Agent Substrate + run: hack/install-ate-kind.sh --deploy-ate-system + - name: Assert the control plane is up + # install-ate.sh runs under pipefail, so a failed apply does propagate. + # What it would not catch is a Deployment that rolls out and then + # crash-loops. Re-check everything deploy_ate_system waits on -- + # atenet-egress included, since a non-dual-stack Envoy listener fails + # there first, by way of a readiness probe the kubelet cannot reach. + run: | + for r in deployment/ate-api-server deployment/ate-controller \ + deployment/atenet-router deployment/atenet-egress \ + deployment/dns statefulset/postgres daemonset/atelet; do + kubectl --context="$KUBECTL_CONTEXT" -n ate-system rollout status "$r" --timeout=120s + done + kubectl --context="$KUBECTL_CONTEXT" -n podcertificate-controller-system \ + rollout status deployment/podcertificate-controller --timeout=120s + if kubectl --context="$KUBECTL_CONTEXT" -n ate-system get pods \ + -o jsonpath='{.items[*].status.containerStatuses[*].state.waiting.reason}' \ + | grep -q CrashLoopBackOff; then + echo "::error::a pod in ate-system is in CrashLoopBackOff" + exit 1 + fi + - name: Assert the node advertises ate.dev/kvm + # A micro-VM WorkerPool requests this extended resource, so an unschedulable + # worker pod is how its absence would otherwise surface -- as a demo deploy + # that hangs until its golden-snapshot deadline. atelet's device plugin + # registers a moment after the DaemonSet reports rolled out, hence the poll. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + for node in $(hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + kvm="" + for _ in $(seq 30); do + n=$(k get node "${node}" -o jsonpath='{.status.allocatable.ate\.dev/kvm}') + case "${n}" in ''|0) sleep 2 ;; *) kvm="${n}"; break ;; esac + done + [ -n "${kvm}" ] || { + echo "::error::${node} does not advertise ate.dev/kvm -- atelet's device plugin never registered it, so no micro-VM worker can be scheduled" + k -n ate-system logs -l app=atelet --tail=50 --prefix || true + exit 1 + } + echo " ${node}: allocatable ate.dev/kvm=${kvm}" + done + - name: Deploy substrate micro-VM counter demo + # Stages the (cached) assets into the cluster's rustfs and applies the + # counter-substrate-microvm demo onto the control plane installed above. + # First of the demos: it is the step a v6-only cluster breaks, and its + # golden snapshot is the long pole, so failing here fails early. The + # substrate demo handler waits for that snapshot itself. + run: | + hack/run-microvm-demo-kind.sh --substrate + df -h / + - name: Deploy substrate gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter-substrate + - name: Deploy egress demos + # One per class: networking builds its egress Actor from the fixture for + # the class under test, so both have to exist before either lane runs. + run: | + hack/install-ate-kind.sh --deploy-demo-egress + hack/install-ate-kind.sh --deploy-demo-egress-microvm + - name: Assert the demo fixtures exist + # A failed demo deploy exits 0: install-ate.sh dispatches demos through + # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends + # in an unconditional `return 0`. Without this the suites fail later with + # "ActorTemplate not found", pointing at the tests instead of the install. + # + # Two lookups, because the two fixtures live in different layers: the + # egress template is a CRD, while the substrate counter's is a store + # object the CRD API never sees. + run: | + for ns_tmpl in ate-demo-egress/egress ate-demo-egress-microvm/egress-microvm; do + ns=${ns_tmpl%/*}; tmpl=${ns_tmpl#*/} + kubectl --context="$KUBECTL_CONTEXT" -n "${ns}" get actortemplate "${tmpl}" \ + || { echo "::error::${ns_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + for as_tmpl in ate-demo-counter-substrate/counter \ + ate-demo-counter-substrate-microvm/counter-microvm; do + as=${as_tmpl%/*}; tmpl=${as_tmpl#*/} + go run ./cmd/kubectl-ate --context="$KUBECTL_CONTEXT" \ + get actor-template "${tmpl}" -a "${as}" \ + || { echo "::error::${as_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + - name: Run E2E tests (networking) + id: e2e-networking + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking.log + - name: Run E2E tests (networking, micro-VM) + # The same suite again, with every fixture repointed at its micro-VM + # variant by the single E2E_SANDBOX_CLASS knob (internal/e2e/sandbox.go). + # Sequential with the gVisor lane above, not concurrent: a suite releases + # its namespace -- and with it its worker pods -- as each test passes, so + # the two runs do not contend for the one kind node. + # + # always(), so a gVisor failure does not hide the micro-VM result. That is + # the whole point of the lane: the two classes reach an actor's addresses + # by different routes and can fail independently. + id: e2e-networking-microvm + if: always() && steps.e2e-networking.outcome != 'skipped' + env: + E2E_SANDBOX_CLASS: microvm + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking-microvm.log + - name: Guard against a vacuously green run + # A test that gates on a dual-stack Service skips itself on v6-only and + # exits 0, so a suite that only skipped would otherwise read as a pass. + # The bar is one real PASS, not zero skips: a test with nothing to compare + # on a single-family cluster is right to skip. Skips are printed so a + # growing list gets noticed. + # Skipped when the suite never ran: with no log to count, this step would + # otherwise report a reassuring zero on a job that failed earlier. + if: always() && steps.e2e-networking.outcome != 'skipped' + env: + MICROVM_RAN: ${{ steps.e2e-networking-microvm.outcome != 'skipped' }} + run: | + logs=/tmp/e2e-networking.log + if [ "${MICROVM_RAN}" = "true" ]; then + logs="${logs} /tmp/e2e-networking-microvm.log" + else + echo "::warning::the micro-VM lane did not run; only gVisor was checked" + fi + for f in ${logs}; do + [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } + passed=$(grep -c -- '--- PASS' "$f" || true) + echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" + grep -h -- '--- SKIP' "$f" || true + if [ "${passed}" -eq 0 ]; then + echo "::error::${f} has no passing tests -- a suite that only skips proves nothing" + exit 1 + fi + done + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context="$KUBECTL_CONTEXT" get actortemplate,workerpool,pods -A -o wide || true + dump() { + echo "=== logs: $1/$2 ===" + kubectl --context="$KUBECTL_CONTEXT" logs -n "$1" "$2" --all-containers --tail=300 2>/dev/null || true + } + for p in $(kubectl --context="$KUBECTL_CONTEXT" get pods -n ate-system -o name 2>/dev/null); do + dump ate-system "$p" + done + # Every worker pod in any namespace: the demo pools plus the e2e suites' + # randomly-named per-test namespaces, which the suites keep on failure. + # The failing actor runs in one of these, so this is where its ateom logs + # -- and, for a micro-VM worker, the guest console tail -- live. + kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ + -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ + | while read -r ns name; do dump "$ns" "$name"; done + # IPv6-specific: the Corefile create-kind-cluster.sh built, and what each + # Service actually got assigned, are the two things that differ from the + # IPv4 job. + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l k8s-app=kube-dns --tail=100 || true + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' || true + kubectl --context="$KUBECTL_CONTEXT" get svc -A \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,POLICY:.spec.ipFamilyPolicy,IPS:.spec.clusterIPs || true + # The agent logs the interface and eBPF setup it did, which is the only + # view of an egress failure that is not a bare timeout. Its pod is the + # one thing here that can be absent on a run that failed early, so say + # so rather than letting kubectl's stderr through: those diagnostics + # most need to not end on something that reads like a second failure. + echo "=== nat64 agent ===" + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get pods -l app=nat64 -o wide \ + 2>/dev/null || echo "(no agent; NAT64 never started)" + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l app=nat64 --tail=100 \ + --prefix 2>/dev/null || true