From 782203c96cd0b67d23800fb5fee791e2935d96b3 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 11:14:17 -0500 Subject: [PATCH] Squash nftables endpoint chains into service vmap We only need a separate chain for the endpoints if the service uses affinity. --- pkg/proxy/nftables/helpers_test.go | 40 +++++- pkg/proxy/nftables/proxier.go | 206 ++++++++++++++--------------- pkg/proxy/nftables/proxier_test.go | 163 ++++++----------------- 3 files changed, 174 insertions(+), 235 deletions(-) diff --git a/pkg/proxy/nftables/helpers_test.go b/pkg/proxy/nftables/helpers_test.go index d2a9c4859b0..daf49b88fd5 100644 --- a/pkg/proxy/nftables/helpers_test.go +++ b/pkg/proxy/nftables/helpers_test.go @@ -270,8 +270,10 @@ var sourceAddrRegexp = regexp.MustCompile(`^ip6* saddr (!= )?(\S+)`) var sourceAddrLookupRegexp = regexp.MustCompile(`^ip6* saddr (!= )?\{([^}]*)\}`) var sourceAddrLocalRegexp = regexp.MustCompile(`^fib saddr type local`) -var endpointVMAPRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) +var endpointVMapRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) var endpointVMapEntryRegexp = regexp.MustCompile(`\d+ : goto (\S+)`) +var endpointDNATRegexp = regexp.MustCompile(`^dnat ip6* addr \. port to numgen random mod \d+ map \{(.*)\}$`) +var endpointDNATEntryRegexp = regexp.MustCompile(`\d+ : (\S+) \. (\d+)`) var masqMarkRegexp = regexp.MustCompile(`^mark set mark or 0x[[:xdigit:]]+$`) var masqCheckRegexp = regexp.MustCompile(`^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+`) @@ -281,7 +283,8 @@ var hairpinCheckRegexp = regexp.MustCompile(`^ip6* saddr \. ip6* daddr \. meta l var jumpRegexp = regexp.MustCompile(`^(jump|goto) (\S+)$`) var returnRegexp = regexp.MustCompile(`^return$`) var verdictRegexp = regexp.MustCompile(`^(drop|reject)$`) -var dnatRegexp = regexp.MustCompile(`^meta l4proto (tcp|udp|sctp) dnat to (\S+)$`) +var l4protoRegexp = regexp.MustCompile(`^meta l4proto (tcp|udp|sctp)`) +var dnatRegexp = regexp.MustCompile(`^dnat to (\S+)$`) var ignoredRegexp = regexp.MustCompile(strings.Join( []string{ @@ -449,6 +452,15 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP break } + case l4protoRegexp.MatchString(rule): + // `meta l4proto (tcp|udp|sctp)` + match := l4protoRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if match[1] != protocol { + rule = "" + break + } + case masqMarkRegexp.MatchString(rule): // `^mark set mark or 0x[[:xdigit:]]+$` // Mark for masquerade. @@ -515,20 +527,20 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP return false case dnatRegexp.MatchString(rule): - // `meta l4proto (tcp|udp|sctp) dnat to (\S+)` + // `^dnat to (\S+)$` // DNAT to an endpoint IP and terminate processing. match := dnatRegexp.FindStringSubmatch(rule) - destEndpoint := match[2] + destEndpoint := match[1] tracer.matches = append(tracer.matches, ruleObj.Rule) tracer.outputs = append(tracer.outputs, destEndpoint) return true - case endpointVMAPRegexp.MatchString(rule): + case endpointVMapRegexp.MatchString(rule): // `^numgen random mod \d+ vmap \{(.*)\}$` // Selects a random endpoint and jumps to it. For tracePacket's // purposes, we jump to *all* of the endpoints. - match := endpointVMAPRegexp.FindStringSubmatch(rule) + match := endpointVMapRegexp.FindStringSubmatch(rule) elements := match[1] for _, match = range endpointVMapEntryRegexp.FindAllStringSubmatch(elements, -1) { @@ -543,6 +555,22 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP } return true + case endpointDNATRegexp.MatchString(rule): + // `^dnat ip6* addr \. port to numgen random mod \d+ map \{(.*)\}$` + // Selects a random endpoint and DNATs to it. For tracePacket's + // purposes, we DNAT to *all* of the endpoints. + match := endpointDNATRegexp.FindStringSubmatch(rule) + elements := match[1] + + for _, match = range endpointDNATEntryRegexp.FindAllStringSubmatch(elements, -1) { + // `\d+ : (\S+) \. (\d+)` + endpointIP, endpointPort := match[1], match[2] + + tracer.matches = append(tracer.matches, ruleObj.Rule) + tracer.outputs = append(tracer.outputs, net.JoinHostPort(endpointIP, endpointPort)) + } + return true + default: tracer.t.Errorf("unmatched rule: %s", ruleObj.Rule) rule = "" diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index 7a57e777862..f8bf741984b 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -155,7 +155,6 @@ type Proxier struct { // updating nftables with some partial data after kube-proxy restart. endpointSlicesSynced bool servicesSynced bool - syncedOnce bool lastFullSync time.Time needFullSync bool initialized int32 @@ -1080,7 +1079,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { doFullSync := proxier.needFullSync || (time.Since(proxier.lastFullSync) > proxyutil.FullSyncPeriod) defer func() { - proxier.syncedOnce = true metrics.SyncProxyRulesLatency.WithLabelValues(string(proxier.ipFamily)).Observe(metrics.SinceInSeconds(start)) if !doFullSync { metrics.SyncPartialProxyRulesLatency.WithLabelValues(string(proxier.ipFamily)).Observe(metrics.SinceInSeconds(start)) @@ -1215,27 +1213,33 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { !serviceUpdateResult.UpdatedServices.Has(svcName.NamespacedName) && !endpointUpdateResult.UpdatedServices.Has(svcName.NamespacedName) - // Note the endpoint chains that will be used - for _, ep := range allLocallyReachableEndpoints { - if epInfo, ok := ep.(*endpointInfo); ok { - ensureChain(epInfo.chainName, tx, activeChains, skipServiceUpdate || - proxier.epChainSkipUpdate(existingChains, existingAffinitySets, svcInfo, epInfo)) - // Note the affinity sets that will be used - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - activeAffinitySets.Insert(epInfo.affinitySetName) - } + // We only use separate endpoint chains when adding affinity rules + serviceUsesAffinity := svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP - // Add endpoint to the hairpin set - proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ - Set: hairpinConnectionsSet, - Key: []string{ - epInfo.IP(), - epInfo.IP(), - protocol, - strconv.Itoa(epInfo.Port()), - }, - }) + // Note the endpoints that will be used + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue } + + // If using affinity, note the endpoint chain and affinity set + // names, and ensure that the chain exists. + if serviceUsesAffinity { + ensureChain(epInfo.chainName, tx, activeChains, skipServiceUpdate) + activeAffinitySets.Insert(epInfo.affinitySetName) + } + + // Add endpoint to the hairpin set + proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ + Set: hairpinConnectionsSet, + Key: []string{ + epInfo.IP(), + epInfo.IP(), + protocol, + strconv.Itoa(epInfo.Port()), + }, + }) } // clusterPolicyChain contains the endpoints used with "Cluster" traffic policy @@ -1624,34 +1628,32 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { } } - // If Cluster policy is in use, create the chain and create rules jumping - // from clusterPolicyChain to the clusterEndpoints - if usesClusterPolicyChain { - proxier.writeServiceToEndpointRules(tx, svcInfo, clusterPolicyChain, clusterEndpoints) - } - - // If Local policy is in use, create rules jumping from localPolicyChain - // to the localEndpoints - if usesLocalPolicyChain { - proxier.writeServiceToEndpointRules(tx, svcInfo, localPolicyChain, localEndpoints) - } - - // Generate the per-endpoint chains - for _, ep := range allLocallyReachableEndpoints { - epInfo, ok := ep.(*endpointInfo) - if !ok { - proxier.logger.Error(nil, "Failed to cast endpointInfo", "endpointInfo", ep) - continue + // Write the endpoint rules and/or chains + if !serviceUsesAffinity { + if usesClusterPolicyChain { + proxier.writeServiceToEndpointDNATs(tx, svcInfo, clusterPolicyChain, clusterEndpoints) + } + if usesLocalPolicyChain { + proxier.writeServiceToEndpointDNATs(tx, svcInfo, localPolicyChain, localEndpoints) + } + } else { + if usesClusterPolicyChain { + proxier.writeServiceToEndpointJumps(tx, svcInfo, clusterPolicyChain, clusterEndpoints) + } + if usesLocalPolicyChain { + proxier.writeServiceToEndpointJumps(tx, svcInfo, localPolicyChain, localEndpoints) } - if proxier.epChainSkipUpdate(existingChains, existingAffinitySets, svcInfo, epInfo) { - // If the EP chain is already updated, we can skip it. - continue - } - endpointChain := epInfo.chainName + // And generate the per-endpoint chains and affinity sets + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } - // Handle session affinity - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { + endpointChain := epInfo.chainName + + // Handle session affinity tx.Add(&knftables.Rule{ Chain: endpointChain, Rule: knftables.Concat( @@ -1659,16 +1661,16 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { "{", ipX, "saddr", "}", ), }) - } - // DNAT to final destination. - tx.Add(&knftables.Rule{ - Chain: endpointChain, - Rule: knftables.Concat( - "meta l4proto", protocol, - "dnat to", epInfo.String(), - ), - }) + // DNAT to final destination. + tx.Add(&knftables.Rule{ + Chain: endpointChain, + Rule: knftables.Concat( + "meta l4proto", protocol, + "dnat to", epInfo.String(), + ), + }) + } } } @@ -1765,55 +1767,12 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { return } -// epChainSkipUpdate returns true if the EP chain doesn't need to be updated. -func (proxier *Proxier) epChainSkipUpdate(existingChains, existingAffinitySets sets.Set[string], svcInfo *servicePortInfo, epInfo *endpointInfo) bool { - if proxier.syncedOnce { - // We only skip updating EP chains during the first sync to speed up kube-proxy restart, otherwise return false. - return false - } - if existingChains == nil || existingAffinitySets == nil { - // listing existing objects failed, can't skip updating - return false - } - // EP chain can have up to 3 rules: - // - loopback masquerade rule - // - includes the endpoint IP - // - affinity rule when session affinity is set to ClusterIP - // - includes the affinity set name - // - DNAT rule - // - includes the endpoint IP + port - // EP chain name includes the endpoint IP + port => loopback and DNAT rules are pre-defined by the chain name. - // When session affinity is set to ClusterIP, the affinity set is created for local endpoints. - // Therefore, we can check that sessions affinity hasn't changed by checking if the affinity set exists. - wantAffinitySet := svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP - return existingChains.Has(epInfo.chainName) && wantAffinitySet == existingAffinitySets.Has(epInfo.affinitySetName) -} - -func (proxier *Proxier) writeServiceToEndpointRules(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { - // First write session affinity rules, if applicable. - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - ipX := "ip" - if proxier.ipFamily == v1.IPv6Protocol { - ipX = "ip6" - } - - for _, ep := range endpoints { - epInfo, ok := ep.(*endpointInfo) - if !ok { - continue - } - - tx.Add(&knftables.Rule{ - Chain: svcChain, - Rule: knftables.Concat( - ipX, "saddr", "@", epInfo.affinitySetName, - "goto", epInfo.chainName, - ), - }) - } +func (proxier *Proxier) writeServiceToEndpointDNATs(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { + ipX := "ip" + if proxier.ipFamily == v1.IPv6Protocol { + ipX = "ip6" } - // Now write loadbalancing rule var elements []string for i, ep := range endpoints { epInfo, ok := ep.(*endpointInfo) @@ -1822,12 +1781,53 @@ func (proxier *Proxier) writeServiceToEndpointRules(tx *knftables.Transaction, s } elements = append(elements, - strconv.Itoa(i), ":", "goto", epInfo.chainName, + strconv.Itoa(i), ":", epInfo.IP(), ".", strconv.Itoa(epInfo.Port()), ) if i != len(endpoints)-1 { elements = append(elements, ",") } } + tx.Add(&knftables.Rule{ + Chain: svcChain, + Rule: knftables.Concat( + "meta l4proto", strings.ToLower(string(svcInfo.Protocol())), + "dnat", ipX, "addr . port to", + "numgen random mod", len(endpoints), "map", + "{", elements, "}", + ), + }) +} + +func (proxier *Proxier) writeServiceToEndpointJumps(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { + ipX := "ip" + if proxier.ipFamily == v1.IPv6Protocol { + ipX = "ip6" + } + + var elements []string + // Write the affinity rules, construct the vmap elements + for i, ep := range endpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } + + tx.Add(&knftables.Rule{ + Chain: svcChain, + Rule: knftables.Concat( + ipX, "saddr", "@", epInfo.affinitySetName, + "goto", epInfo.chainName, + ), + }) + + elements = append(elements, + strconv.Itoa(i), ":", "goto", epInfo.chainName, + ) + if i != len(endpoints)-1 { + elements = append(elements, ",") + } + } + // Now write the vmap, for the case where no affinity rule matched tx.Add(&knftables.Rule{ Chain: svcChain, Rule: knftables.Concat( diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index a473ee704a6..18fa2cc1140 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -366,10 +366,7 @@ func TestOverallNFTablesRules(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } - - add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.1 . 80 } add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } @@ -377,13 +374,11 @@ func TestOverallNFTablesRules(t *testing.T) { # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 - add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 } + add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.2 . 80 } add chain ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 ip saddr 10.0.0.0/8 goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit pod traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local mark set mark or 0x4000 comment "masquerade local traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit local traffic" - add chain ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 - add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 meta l4proto tcp dnat to 10.180.0.2:80 add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 80 : goto service-42NFTM6N-ns2/svc2/tcp/p80 } @@ -398,12 +393,10 @@ func TestOverallNFTablesRules(t *testing.T) { # svc3 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 } + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.3 . 80 } add chain ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 goto service-4AT6LBPK-ns3/svc3/tcp/p80 - add chain ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } @@ -412,14 +405,10 @@ func TestOverallNFTablesRules(t *testing.T) { # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 , 1 : goto endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 } + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.180.0.4 . 80 , 1 : 10.180.0.5 . 80 } add chain ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 goto service-LAUZTJTB-ns4/svc4/tcp/p80 - add chain ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 - add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 meta l4proto tcp dnat to 10.180.0.5:80 - add chain ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 - add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 meta l4proto tcp dnat to 10.180.0.4:80 add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } @@ -3894,14 +3883,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -3946,25 +3931,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 3, // add 1 element each to cluster-ips, service-ips, and hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule ) // Delete a service; its chains will be flushed, but not immediately deleted. @@ -3979,22 +3957,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 3, // delete 1 element each from cluster-ips, service-ips, hairpin-connections - 2, // flush 2 chains for service and endpoint + 1, // flush service chain ) // Fake the passage of time and confirm that the stale chains get deleted. @@ -4009,14 +3982,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // delete stale chains happens in a separate transaction, nothing else changed @@ -4045,14 +4014,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } `) @@ -4087,26 +4052,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule ) // Change an endpoint of an existing service. @@ -4128,27 +4086,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.2 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 old element from hairpin-connections, add 1 new one 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule - 1, // flush old endpoint chain ) // (Ensure the old svc3 chain gets deleted in the next sync.) @@ -4173,29 +4122,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.3.2 . 80 , 1 : 10.0.3.3 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush original endpoint chain, add 1 rule - 3, // add+flush new endpoint chain, add 1 rule ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. @@ -4214,27 +4153,21 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services 2, // remove 2 elements from hairpin-connections - 3, // flush 3 chains + 1, // flush service chain ) - expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80", "endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80", "endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80") + expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80") gotStaleChains := sets.StringKeySet(fp.staleChains) if !expectedStaleChains.Equal(gotStaleChains) { t.Errorf("expected stale chains %v, got %v", expectedStaleChains, gotStaleChains) @@ -4255,28 +4188,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.3.2 . 80 , 1 : 10.0.3.3 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips 2, // add 2 elements to hairpin-connections 3, // add+flush service chain, add 1 rule - 6, // add+flush 2 endpoint chains, add 1 rule to each ) if len(fp.staleChains) != 0 { @@ -4312,15 +4236,11 @@ func TestSyncProxyRulesStartup(t *testing.T) { // put a part of desired state to nftables err := nft.ParseDump(baseRules + dedent.Dedent(` add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy cluster-ips { 172.30.0.42 } @@ -4407,16 +4327,10 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 - add chain ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 - add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.1.1 . 80 , 1 : 10.0.1.2 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, @@ -4424,7 +4338,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { 1, // add new svc1 endpoint to hairpin-connections 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set 6, // add+flush 2 service chains + 1 rule each - 3, // add+flush svc1 endpoint chain + 1 rule ) } @@ -4881,14 +4794,12 @@ func TestBadIPs(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.1 . 80 } add chain ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 goto service-ULMVA6XW-ns1/svc1/tcp/p80 - add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add chain ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 ip saddr != { 203.0.113.0/25 } drop