Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
8cd2a01
hack: drop the IPv6 kubeconfig repoint
ygao-g Aug 27, 2026
2456710
hack: fix DNS on IPv6-only kind clusters
ygao-g Aug 27, 2026
4d00b5e
hack: add opt-in NAT64 support to IPv6-only kind clusters
ygao-g Aug 27, 2026
00c5406
atunnel: support IPv6 original destination lookup
lubingtan Aug 5, 2026
f3f3814
atunnel: preserve IPv4 original destination errors
lubingtan Aug 21, 2026
ef064bb
atunnel: stabilize original destination tests
lubingtan Aug 21, 2026
a9bade0
atunnel: select original destination by family
lubingtan Aug 24, 2026
af37c2a
atunnel: identify original destination family in errors
lubingtan Aug 24, 2026
5506f7a
atunnel: isolate original destination tests
lubingtan Aug 24, 2026
d1469ad
atunnel: test original destination formatting
lubingtan Aug 24, 2026
cf0fbcd
atunnel: document original destination buffer sizes
lubingtan Aug 24, 2026
404ec60
ateom: drop the family from the atunnel ingress listen defaults
ygao-g Aug 19, 2026
b3ef35f
atecontroller: drop the family from the atunnel ingress args
ygao-g Aug 20, 2026
fcb1a89
ateom: fix the comment about which listen addresses are dual-stack
ygao-g Aug 20, 2026
606825e
ateomnet: enable IPv6 forwarding in worker pod netns
krsnaSuraj Aug 15, 2026
1cd4703
ateomnet: return nil when sysctl path missing in writeSysctlIfUnset
krsnaSuraj Aug 16, 2026
b357f52
ateomnet: rename EnableIPv4Forwarding to EnableForwarding
krsnaSuraj Aug 21, 2026
a67785c
ateomnet: cover writeSysctlIfUnset's missing-path branch
krsnaSuraj Aug 21, 2026
c501a6e
atenet/egress: resolve upstream names on both address families
ygao-g Aug 20, 2026
aeb1b7c
ci: add an IPv6-only kind e2e job
ygao-g Aug 14, 2026
47655ec
ateomnet: move the actor nftables table to the inet family
ygao-g Aug 20, 2026
703b8ee
ateomnet: give the actor an IPv6 address when the pod has one
ygao-g Aug 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
370 changes: 370 additions & 0 deletions .github/workflows/e2e-ipv6.yaml

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions cmd/atecontroller/internal/controllers/workerpool_apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin
WithImage(wp.Spec.AteomImage).
WithArgs(
"--pod-uid=$(POD_UID)",
"--atunnel-listen-address=0.0.0.0:443",
"--atunnel-connect-listen-address=0.0.0.0:444",
"--atunnel-listen-address=:443",
"--atunnel-connect-listen-address=:444",
"--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem",
"--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem",
"--atunnel-egress-listen-address=0.0.0.0:15001",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -849,8 +849,8 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf
WithImage(wp.Spec.AteomImage).
WithArgs(
"--pod-uid=$(POD_UID)",
"--atunnel-listen-address=0.0.0.0:443",
"--atunnel-connect-listen-address=0.0.0.0:444",
"--atunnel-listen-address=:443",
"--atunnel-connect-listen-address=:444",
"--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem",
"--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem",
"--atunnel-egress-listen-address=0.0.0.0:15001",
Expand Down
136 changes: 136 additions & 0 deletions cmd/atenet/internal/router/extproc/egress_manifest_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package extproc

import (
"bufio"
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"

k8syaml "k8s.io/apimachinery/pkg/util/yaml"
"sigs.k8s.io/yaml"
)

// The install tree, not a fixture, so this guards the Envoy config that ships.
const manifestsDir = "../../../../../manifests"

// What the install ships today. Falling below it means the walk stopped
// matching, not that the config got better.
const minDNSCacheConfigs = 7

// TestEgressDNSLookupFamily requires ALL on every dynamic forward proxy DNS
// cache the install ships; atenet-egress.yaml says why ALL.
func TestEgressDNSLookupFamily(t *testing.T) {
found := 0
for _, path := range manifestPaths(t) {
caches := dnsCacheConfigs(t, path)
if len(caches) == 0 {
continue
}
found += len(caches)
t.Run(filepath.Base(path), func(t *testing.T) {
for _, cache := range caches {
if got := cache["dns_lookup_family"]; got != "ALL" {
t.Errorf("dns_cache_config %v: dns_lookup_family = %v, want ALL", cache["name"], got)
}
}
})
}
if found < minDNSCacheConfigs {
t.Errorf("found %d dns_cache_config blocks under %s, want at least %d", found, manifestsDir, minDNSCacheConfigs)
}
}

// manifestPaths covers the whole install tree, so a new egress variant is
// checked the day it is added.
func manifestPaths(t *testing.T) []string {
t.Helper()
var paths []string
err := filepath.WalkDir(manifestsDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) {
paths = append(paths, path)
}
return nil
})
if err != nil {
t.Fatalf("walking %s: %v", manifestsDir, err)
}
return paths
}

func dnsCacheConfigs(t *testing.T, path string) []map[string]any {
t.Helper()
f, err := os.Open(path)
if err != nil {
t.Fatalf("opening %s: %v", path, err)
}
defer f.Close()

var caches []map[string]any
reader := k8syaml.NewYAMLReader(bufio.NewReader(f))
for {
doc, err := reader.Read()
if errors.Is(err, io.EOF) {
return caches
}
if err != nil {
t.Fatalf("reading %s: %v", path, err)
}
var object struct {
Kind string `json:"kind"`
Data map[string]string `json:"data"`
}
if err := yaml.Unmarshal(doc, &object); err != nil {
t.Fatalf("parsing a document of %s: %v", path, err)
}
if object.Kind != "ConfigMap" {
continue
}
for _, value := range object.Data {
var config any
if err := yaml.Unmarshal([]byte(value), &config); err != nil {
// Not every ConfigMap value is YAML.
continue
}
caches = append(caches, collectDNSCacheConfigs(config)...)
}
}
}

func collectDNSCacheConfigs(node any) []map[string]any {
var caches []map[string]any
switch node := node.(type) {
case map[string]any:
for key, value := range node {
if cache, ok := value.(map[string]any); ok && key == "dns_cache_config" {
caches = append(caches, cache)
}
caches = append(caches, collectDNSCacheConfigs(value)...)
}
case []any:
for _, value := range node {
caches = append(caches, collectDNSCacheConfigs(value)...)
}
}
return caches
}
7 changes: 5 additions & 2 deletions cmd/ateom-gvisor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,11 @@ var (
podUID = pflag.String("pod-uid", "", "The UID of the current pod")

// TODO(liorlieberman) have a sub package for all atunnel releated things like that
atunnelListenAddress = pflag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS")
atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", "0.0.0.0:444", "Address for actor ingress mTLS CONNECT")

// Every listen address here is an unspecified wildcard, which Go binds as a
// dual-stack socket.
atunnelListenAddress = pflag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS")
atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT")
workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS")
podIdentityTrustBundle = pflag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet")
atunnelClientIdentity = pflag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS")
Expand Down
6 changes: 4 additions & 2 deletions cmd/ateom-microvm/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,10 @@ var (
otlpRelaySocket = flag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(),
"Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.")

atunnelListenAddress = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS")
atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", "0.0.0.0:444", "Address for actor ingress mTLS CONNECT")
// Every listen address here is an unspecified wildcard, which Go binds as a
// dual-stack socket.
atunnelListenAddress = flag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS")
atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT")
workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS")
podIdentityTrustBundle = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet")
atunnelClientIdentity = flag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS")
Expand Down
4 changes: 4 additions & 0 deletions cmd/ateom-microvm/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -1042,6 +1042,10 @@ func tailString(s string, n int) string {
// agent: configure eth0 (IP/MAC/MTU), install the connected + default routes, and
// pin the gateway's ARP entry to its fixed MAC (so a restored guest's frozen
// neighbor entry stays valid).
//
// TODO(#246): the guest is configured IPv4-only, so a micro-VM actor sees no
// IPv6 even on a dual-stack pod where the host veth has one. gVisor reads the
// interior netns and picks the address up; this path has to be told.
func (s *AteomService) configureGuestNetwork(ctx context.Context, ac *kata.AgentClient, mtu uint64) error {
if err := ac.UpdateInterface(ctx, &agentpb.Interface{
Device: ateomnet.ActorVethName,
Expand Down
142 changes: 127 additions & 15 deletions hack/create-kind-cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}"
KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}"
reg_name="kind-registry"
reg_port="${KIND_REGISTRY_PORT:-5001}"
IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}"
IPV6_DNS64_PREFIX="${IPV6_DNS64_PREFIX:-}"

if [[ $# -gt 0 ]]; then
case "$1" in
Expand All @@ -31,6 +33,14 @@ if [[ $# -gt 0 ]]; then
echo "Configured through the environment:"
echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)."
echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)."
echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6"
echo " (default: Google Public DNS). These replace the host's resolver, so any"
echo " split-horizon names it served stop resolving from pods."
echo " IPV6_DNS64_PREFIX NAT64 prefix cluster DNS synthesizes external names into when"
echo " IP_FAMILY=ipv6, e.g. 64:ff9b::/96 (default: empty, no DNS64)."
echo " Set it only on a host with no IPv6 egress, together with"
echo " hack/setup-nat64.sh: it routes every external name through the"
echo " prefix, including names that already have reachable AAAA records."
exit 0
;;
esac
Expand Down Expand Up @@ -135,6 +145,23 @@ fi
echo "Creating kind cluster '${KIND_CLUSTER_NAME}'..."
"${ROOT}"/hack/kind.sh create cluster --name "${KIND_CLUSTER_NAME}" --config "${ROOT}/bin/kind-config.yaml"

# kind create returns before the apiserver answers, and every kubectl below races
# it. Poll from inside the node, where the answer does not depend on how the
# daemon published the port.
echo "Waiting for the control plane to answer..."
for attempt in $(seq 60); do
if docker exec "${KIND_CLUSTER_NAME}-control-plane" \
kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw /healthz >/dev/null 2>&1; then
break
fi
if [[ "${attempt}" == 60 ]]; then
echo "error: the control plane did not answer /healthz within 2m of create:" >&2
echo " docker logs ${KIND_CLUSTER_NAME}-control-plane" >&2
exit 1
fi
sleep 2
done

# A daemon with IPv6 off hands kind a v4-only network whatever it asked for.
if [[ "${IP_FAMILY}" != "ipv4" &&
"$(docker network inspect kind --format '{{.EnableIPv6}}')" != "true" ]]; then
Expand All @@ -145,21 +172,6 @@ if [[ "${IP_FAMILY}" != "ipv4" &&
exit 1
fi

# For ipv6 kind writes a kubeconfig pointing at [::1], the address it published
# the apiserver on, which only works for a client on the Docker host itself: a
# VM-hosted daemon (Lima on macOS) forwards the port to the *v4* loopback, so
# every kubectl below fails at connect. localhost is a SAN on the apiserver
# cert and lets the client pick a family that works from either side.
if [[ "${IP_FAMILY}" == "ipv6" ]]; then
server="$(kubectl config view \
-o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")"
if [[ "${server}" == "https://[::1]:"* ]]; then
echo "Repointing the kubeconfig for '${KUBECTL_CONTEXT}' at localhost..."
kubectl config set-cluster "${KUBECTL_CONTEXT}" \
--server="https://localhost:${server##*:}" >/dev/null
fi
fi

# 2.5 Enable Proxy ARP/NDP on kind nodes for gVisor loopback pod-to-pod networking
echo "Enabling Proxy ARP/NDP on kind nodes..."
for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do
Expand Down Expand Up @@ -196,6 +208,106 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}
docker network connect "kind" "${reg_name}"
fi

# 4.5. Point CoreDNS at an IPv6 resolver and teach it the registry's name
if [[ "${IP_FAMILY}" == "ipv6" ]]; then
echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..."
reg_v6="$(docker inspect "${reg_name}" \
--format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)"
if [[ -z "${reg_v6}" ]]; then
echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2
exit 1
fi

# CoreDNS runs dnsPolicy: Default and inherits the node's IPv4 resolver, which
# no pod here can reach.
corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \
-o jsonpath='{.data.Corefile}')"
search="forward . /etc/resolv.conf"
# $search unquoted: bash 3.2 splices the quotes in literally.
patched="${corefile/$search/forward . ${IPV6_DNS_UPSTREAM}}"
if [[ "${patched}" == "${corefile}" ]]; then
echo "error: '${search}' not found in the CoreDNS Corefile" >&2
echo " the Corefile layout changed upstream; update this block" >&2
exit 1
fi

# Step 3's registry wiring is node-side, while atelet pulls from its own netns,
# where "kind-registry" does not resolve. Own zone, so no fallthrough is needed:
# only this name reaches the hosts stanza.
patched="${patched}
${reg_name}:53 {
errors
hosts {
${reg_v6} ${reg_name}
}
}"

if [[ -n "${IPV6_DNS64_PREFIX}" ]]; then
echo "Synthesizing external names into ${IPV6_DNS64_PREFIX}..."
# Plain DNS64 synthesizes only for names with no AAAA, and the names that
# matter here have real AAAA records pointing at addresses a host with no
# IPv6 egress cannot reach. Only translate_all forces them through the
# prefix -- which is why this is opt-in: where IPv6 egress does work it
# replaces reachable answers with unreachable ones.
#
# translate_all cannot share a server block with the cluster zones. dns64
# wraps the plugin chain below it and answers AAAA by synthesizing from A,
# so for an AAAA-only name it synthesizes from nothing and returns an empty
# answer. Every ClusterIP here is AAAA-only, so one block would take out all
# in-cluster service discovery. Re-zone what kind shipped to the cluster
# zones, lift its forwarder out, and give dns64 the catch-all.
#
# The prefix goes inside the block, not on the `dns64` line: CoreDNS takes
# `dns64 PREFIX { ... }` without complaint and then never applies the block,
# so translate_all silently does nothing and only AAAA-less names get
# synthesized -- which looks like it works until something with a real AAAA
# is the thing that has to be reached.
rezoned="$(printf '%s\n' "${patched}" | awk '
NR == 1 && /^\.:53[[:space:]]*\{/ {
print "cluster.local:53 in-addr.arpa:53 ip6.arpa:53 {"; first = 1; next
}
first && /^ forward([[:space:]].*)?\{$/ { skip = 1; next }
first && skip && /^ \}$/ { skip = 0; next }
first && skip { next }
first && /^\}$/ { first = 0 }
{ print }
')"
case "${rezoned}" in
"cluster.local:53"*) ;;
*) echo "error: the Corefile does not open with the '.:53' block kind ships" >&2
echo " the Corefile layout changed upstream; update this block" >&2
exit 1 ;;
esac
if printf '%s' "${rezoned}" | grep -q 'forward'; then
echo "error: a forward block survived the re-zone" >&2
exit 1
fi
patched="${rezoned}
.:53 {
errors
dns64 {
prefix ${IPV6_DNS64_PREFIX}
translate_all
}
forward . ${IPV6_DNS_UPSTREAM} {
max_concurrent 1000
}
cache 30
loop
reload
}"
fi

# A YAML patch file avoids escaping the Corefile's newlines into JSON.
{ printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \
> "${ROOT}/bin/coredns-patch.yaml"
kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \
--type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml"
kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns
kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \
--timeout=120s
fi

# 5. Document the local registry in kube-public ConfigMap
echo "Documenting local registry in cluster..."
cat <<EOF | kubectl --context="${KUBECTL_CONTEXT}" apply -f -
Expand Down
Loading
Loading