From cfcb10ef8acdb0305b703002b00e117a71814e2e Mon Sep 17 00:00:00 2001
From: Davanum Srinivas <davanum@gmail.com>
Date: Fri, 30 Jan 2026 08:17:06 -0500
Subject: [PATCH] hack: switch local-up-cluster to ptp CNI for reliable DIND
 networking

The ci-kubernetes-local-e2e job has been flaky (~40-45% success rate)
with intermittent DNS/service connectivity failures. The root cause is
that bridge CNI requires br_netfilter and bridge-nf-call-iptables
kernel settings, which don't work reliably in docker-in-docker.

This switches to ptp (point-to-point) CNI, which creates direct veth
pairs between pods and host namespace. No bridge means no br_netfilter
dependency. This is the same approach KIND uses and it works reliably.

Changes:
- Replace bridge CNI with ptp CNI plugin
- Configure kernel network parameters for DIND (route_localnet,
  arp_ignore, ip_forward) required for ptp and iptables-based kube-proxy
- Remove CoreDNS pod delete/restart workaround from 1168b11875 that was
  masking the underlying networking issues (no longer needed)
- Add CoreDNS log capture during cleanup for debugging DNS issues

Signed-off-by: Davanum Srinivas <davanum@gmail.com>
---
 hack/local-up-cluster.sh | 64 ++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh
index 741435087fb..b5e7399d66a 100755
--- a/hack/local-up-cluster.sh
+++ b/hack/local-up-cluster.sh
@@ -424,6 +424,13 @@ function detect_binary {
 cleanup()
 {
   echo "Cleaning up..."
+
+  # Capture CoreDNS logs before shutting down (useful for debugging DNS issues)
+  if [[ "${ENABLE_CLUSTER_DNS}" == true ]]; then
+    echo "Capturing CoreDNS logs..."
+    ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" logs -n kube-system -l k8s-app=kube-dns --all-containers > "${LOG_DIR}/coredns.log" 2>&1 || true
+  fi
+
   # delete running images
   # if [[ "${ENABLE_CLUSTER_DNS}" == true ]]; then
   # Still need to figure why this commands throw an error: Error from server: client: etcd cluster is unavailable or misconfigured
@@ -841,27 +848,6 @@ function wait_coredns_available(){
     return
   fi
 
-  local interval_time=2
-  local coredns_wait_time=300
-
-  # kick the coredns pods to be recreated
-  ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" -n kube-system delete pods -l k8s-app=kube-dns
-  sleep 30
-
-  local coredns_pods_ready="${KUBECTL} --kubeconfig '${CERT_DIR}/admin.kubeconfig' wait --for=condition=Ready --timeout=60s pods -l k8s-app=kube-dns -n kube-system"
-  kube::util::wait_for_success "$coredns_wait_time" "$interval_time" "$coredns_pods_ready"
-  if [ $? == "1" ]; then
-    echo "time out on waiting for coredns pods"
-    exit 1
-  fi
-
-  local coredns_available="${KUBECTL} --kubeconfig '${CERT_DIR}/admin.kubeconfig' wait --for=condition=Available --timeout=60s deployments coredns -n kube-system"
-  kube::util::wait_for_success "$coredns_wait_time" "$interval_time" "$coredns_available"
-  if [ $? == "1" ]; then
-    echo "time out on waiting for coredns deployment"
-    exit 1
-  fi
-
   if [[ "${ENABLE_DAEMON}" = false ]]; then
     # bump log level
     echo "6" | sudo tee /proc/sys/kernel/printk
@@ -1312,14 +1298,19 @@ function install_cni {
     rm -rf "${cni_plugin_tarball}" &&
     sudo find /opt/cni/bin -type f -not \( \
         -iname host-local \
-        -o -iname bridge \
+        -o -iname ptp \
         -o -iname portmap \
         -o -iname loopback \
         \) \
         -delete
 
-  # containerd in kubekins supports CNI version 0.4.0
-  echo "Configuring cni"
+  # Configure CNI using ptp (point-to-point) plugin instead of bridge.
+  # ptp creates direct veth pairs between pods and host namespace, which
+  # avoids the need for br_netfilter and bridge-nf-call-iptables settings
+  # that are unreliable in docker-in-docker environments.
+  # This approach is proven to work reliably by KIND (Kubernetes IN Docker):
+  # https://github.com/kubernetes-sigs/kind/blob/main/images/kindnetd/cmd/kindnetd/cni.go#L83-L121
+  echo "Configuring cni (ptp mode)"
   sudo mkdir -p "$CNI_CONFIG_DIR"
   cat << EOF | sudo tee "$CNI_CONFIG_DIR"/10-containerd-net.conflist
 {
@@ -1327,11 +1318,8 @@ function install_cni {
  "name": "containerd-net",
  "plugins": [
    {
-     "type": "bridge",
-     "bridge": "cni0",
-     "isGateway": true,
+     "type": "ptp",
      "ipMasq": true,
-     "promiscMode": true,
      "ipam": {
        "type": "host-local",
        "ranges": [
@@ -1387,6 +1375,26 @@ if [[ "${KUBETEST_IN_DOCKER:-}" == "true" ]]; then
   # configure shared mounts to prevent failure in DIND scenarios
   mount --make-rshared /
 
+  # Configure kernel network parameters for container networking.
+  # These settings are required for ptp CNI and iptables-based kube-proxy
+  # to work correctly in docker-in-docker environments.
+  # See KIND's network configuration for reference.
+  echo "Configuring kernel network parameters for DIND..."
+
+  # Enable route_localnet - allows routing to localhost addresses after NAT.
+  # Required for proper DNS resolution in containers.
+  echo 1 > /proc/sys/net/ipv4/conf/all/route_localnet
+
+  # Set arp_ignore=0 - required for ptp CNI which uses /32 addresses.
+  # Ensures ARP replies are sent for all local addresses.
+  echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
+
+  # Ensure IP forwarding is enabled for pod-to-pod traffic
+  echo 1 > /proc/sys/net/ipv4/ip_forward
+  echo 1 > /proc/sys/net/ipv6/conf/all/forwarding 2>/dev/null || true
+
+  echo "Kernel network parameters configured"
+
   # to use containerd as kubelet container runtime we need to install cni
   install_cni