From 3c1ad42773d325295d82fcf545ea1cb707c7fc2b Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 11 Dec 2025 15:51:20 -0500 Subject: [PATCH 1/4] Add a helper for the operation-counting unit tests in nftables (This will make it easier to keep the counts in sync when we change things.) --- pkg/proxy/nftables/proxier_test.go | 118 +++++++++++++++-------------- 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index db2b4a1bce3..4b64d22ffda 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -3808,6 +3808,20 @@ func TestInternalExternalMasquerade(t *testing.T) { } } +// assertNumOperations asserts that the last transaction on nft had a number of operations +// equal to the sum of the values in ops[]. (ops is multiple values just for ease of +// explaining the math; see the callers below.) +func assertNumOperations(t *testing.T, nft *knftables.Fake, ops ...int) { + t.Helper() + expectedOps := 0 + for _, n := range ops { + expectedOps += n + } + if nft.LastTransaction.NumOperations() != expectedOps { + t.Errorf("Expected transaction with %d operations, got %d", expectedOps, nft.LastTransaction.NumOperations()) + } +} + // Test calling syncProxyRules() multiple times with various changes func TestSyncProxyRulesRepeated(t *testing.T) { nft, fp := NewFakeProxier(v1.IPv4Protocol) @@ -3953,12 +3967,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add 1 element to cluster-ips and service-ips = 2 operations - // add+flush 2 chains for service and endpoint, add 2 rules in each = 8 operations - // 10 operations total. - if nft.LastTransaction.NumOperations() != 10 { - t.Errorf("Expected 10 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // add 1 element each to cluster-ips and service-ips + 4, // add+flush service chain, add 2 rules + 4, // add+flush endpoint chain, add 2 rules + ) // Delete a service; its chains will be flushed, but not immediately deleted. fp.OnServiceDelete(svc2) @@ -3987,12 +4000,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // delete 1 element from cluster-ips and service-ips = 2 operations - // flush 2 chains for service and endpoint = 2 operations - // 4 operations total. - if nft.LastTransaction.NumOperations() != 4 { - t.Errorf("Expected 4 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // delete 1 element each from cluster-ips and service-ips + 2, // flush 2 chains for service and endpoint + ) // Fake the passage of time and confirm that the stale chains get deleted. ageStaleChains() @@ -4018,10 +4029,8 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // delete stale chains happens in a separate transaction, nothing else changed => last transaction will have 0 operations. - if nft.LastTransaction.NumOperations() != 0 { - t.Errorf("Expected 0 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + // delete stale chains happens in a separate transaction, nothing else changed + assertNumOperations(t, nft, 0) // Add a service, sync, then add its endpoints. makeServiceMap(fp, @@ -4060,10 +4069,9 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add 1 element to cluster-ips and no-endpoint-services = 2 operations - if nft.LastTransaction.NumOperations() != 2 { - t.Errorf("Expected 2 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // add 1 element each to cluster-ips and no-endpoint-services + ) populateEndpointSlices(fp, makeTestEndpointSlice("ns4", "svc4", 1, func(eps *discovery.EndpointSlice) { @@ -4109,11 +4117,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add 1 element to service-ips, remove 1 element from no-endpoint-services = 2 operations - // add+flush 2 chains for service and endpoint, add 2 rules in each = 8 operations - if nft.LastTransaction.NumOperations() != 10 { - t.Errorf("Expected 10 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services + 4, // add+flush service chain, add 2 rules + 4, // add+flush endpoint chain, add 2 rules + ) // Change an endpoint of an existing service. eps3update := eps3.DeepCopy() @@ -4153,11 +4161,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add+flush 2 chains for service and endpoint, add 2 rules in each = 8 operations - // flush old endpoint chain = 1 operation - if nft.LastTransaction.NumOperations() != 9 { - t.Errorf("Expected 9 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 4, // add+flush service chain, add 2 rules + 4, // add+flush endpoint chain, add 2 rules + 1, // flush old endpoint chain + ) // (Ensure the old svc3 chain gets deleted in the next sync.) ageStaleChains() @@ -4201,10 +4209,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // add+flush 3 chains for 1 service and 2 endpoints, add 2 rules in each = 12 operations - if nft.LastTransaction.NumOperations() != 12 { - t.Errorf("Expected 12 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + // (The first endpoint chain is unchanged, but the code recreates it anyway.) + assertNumOperations(t, nft, + 4, // add+flush service chain, add 2 rules + 4, // add+flush original endpoint chain, add 2 rules + 4, // add+flush new endpoint chain, add 2 rules + ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. eps3update3 := eps3update2.DeepCopy() @@ -4238,11 +4248,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // remove 1 element from service-ips, add 1 element to no-endpoint-services = 2 operations - // flush 3 chains = 3 operations - if nft.LastTransaction.NumOperations() != 5 { - t.Errorf("Expected 5 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services + 3, // flush 3 chains + ) expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80", "endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80", "endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80") gotStaleChains := sets.StringKeySet(fp.staleChains) @@ -4285,11 +4294,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // remove 1 element from no-endpoint-services, add 1 element to service-ips = 2 operations - // add+flush 3 chains for 1 service and 2 endpoints, add 2 rules in each = 12 operations - if nft.LastTransaction.NumOperations() != 14 { - t.Errorf("Expected 14 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips + 4, // add+flush service chain, add 2 rules + 8, // add+flush 2 endpoint chains, add 2 rules to each + ) if len(fp.staleChains) != 0 { t.Errorf("unexpected stale chains: %v", fp.staleChains) @@ -4309,9 +4318,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { // Sync with no new changes, so same expected rules as last time fp.syncProxyRules() assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - if nft.LastTransaction.NumOperations() != 0 { - t.Errorf("Expected 0 trasaction operations, got %d", nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, 0) } func TestSyncProxyRulesStartup(t *testing.T) { @@ -4437,17 +4444,12 @@ func TestSyncProxyRulesStartup(t *testing.T) { add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) - // initial transaction consists of: - // 1. nft setup, total ops = setupOps - // 2. services setup (should skip adding existing set/map elements and endpoint chains+rules) - // - add svc3 IP to the cluster-ips, and to the no-endpoint-services set = 2 ops - // - add+flush 2 service chains + 2 rules each = 8 ops - // - add+flush svc1 endpoint chain + 2 rules = 4 ops - // total: 14 ops - if nft.LastTransaction.NumOperations() != setupOps+14 { - fmt.Println(nft.LastTransaction) - t.Errorf("Expected %v trasaction operations, got %d", setupOps+14, nft.LastTransaction.NumOperations()) - } + assertNumOperations(t, nft, + setupOps, // nft setup + 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set + 8, // add+flush 2 service chains + 2 rules each + 4, // add+flush svc1 endpoint chain + 2 rules + ) } func TestNoEndpointsMetric(t *testing.T) { From 6eee34094c6be1668baa8d04392ca2aead81845f Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 14:02:25 -0500 Subject: [PATCH 2/4] Do clusterIP masquerading centrally rather than per-service --- pkg/proxy/nftables/proxier.go | 56 ++++++++++++++---------------- pkg/proxy/nftables/proxier_test.go | 48 ++++--------------------- 2 files changed, 34 insertions(+), 70 deletions(-) diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index e975c791ec8..967c2849e41 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -614,6 +614,33 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { Type: "inet_proto . inet_service : verdict", Comment: ptr.To("NodePort traffic"), }) + + if proxier.masqueradeAll { + tx.Add(&knftables.Rule{ + Chain: servicesChain, + Rule: knftables.Concat( + ipX, "daddr", "@", clusterIPsSet, + proxier.masqueradeRule, + ), + Comment: ptr.To("masquerade all clusterIP traffic"), + }) + } else if proxier.localDetector.IsImplemented() { + // This masquerades off-cluster traffic to a service VIP. The + // idea is that you can establish a static route for your + // Service range, routing to any node, and that node will + // bridge into the Service for you. Since that might bounce + // off-node, we masquerade here. + tx.Add(&knftables.Rule{ + Chain: servicesChain, + Rule: knftables.Concat( + ipX, "daddr", "@", clusterIPsSet, + proxier.localDetector.IfNotLocalNFT(), + proxier.masqueradeRule, + ), + Comment: ptr.To("masquerade clusterIP traffic from outside cluster"), + }) + } + tx.Add(&knftables.Rule{ Chain: servicesChain, Rule: knftables.Concat( @@ -1430,35 +1457,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { continue } - // Set up internal traffic handling. - if hasInternalEndpoints { - if proxier.masqueradeAll { - tx.Add(&knftables.Rule{ - Chain: internalTrafficChain, - Rule: knftables.Concat( - ipX, "daddr", svcInfo.ClusterIP(), - proxier.masqueradeRule, - ), - Comment: ptr.To("masquerade all service traffic"), - }) - } else if proxier.localDetector.IsImplemented() { - // This masquerades off-cluster traffic to a service VIP. The - // idea is that you can establish a static route for your - // Service range, routing to any node, and that node will - // bridge into the Service for you. Since that might bounce - // off-node, we masquerade here. - tx.Add(&knftables.Rule{ - Chain: internalTrafficChain, - Rule: knftables.Concat( - ipX, "daddr", svcInfo.ClusterIP(), - proxier.localDetector.IfNotLocalNFT(), - proxier.masqueradeRule, - ), - Comment: ptr.To("masquerade traffic from outside cluster"), - }) - } - } - // Set up external traffic handling (if any "external" destinations are // enabled). All captured traffic for all external destinations should // jump to externalTrafficChain, which will handle some special cases and diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index 4b64d22ffda..ad991815406 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -188,6 +188,7 @@ var baseRules = dedent.Dedent(` add rule ip kube-proxy nat-prerouting jump services add rule ip kube-proxy nodeport-endpoints-check ip daddr @nodeport-ips meta l4proto . th dport vmap @no-endpoint-nodeports add rule ip kube-proxy reject-chain reject + add rule ip kube-proxy services ip daddr @cluster-ips ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade clusterIP traffic from outside cluster" add rule ip kube-proxy services ip daddr . meta l4proto . th dport vmap @service-ips add rule ip kube-proxy services ip daddr @nodeport-ips meta l4proto . th dport vmap @service-nodeports @@ -362,7 +363,6 @@ func TestOverallNFTablesRules(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 @@ -374,7 +374,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 - add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 } add chain ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 ip saddr 10.0.0.0/8 goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit pod traffic" @@ -396,7 +395,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc3 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 } add chain ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" @@ -411,7 +409,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 , 1 : goto endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 } add chain ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" @@ -430,7 +427,6 @@ func TestOverallNFTablesRules(t *testing.T) { # svc5 add set ip kube-proxy affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 { type ipv4_addr ; flags dynamic,timeout ; timeout 10800s ; } add chain ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 - add rule ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 ip daddr 172.30.0.45 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 ip saddr @affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 goto endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 add rule ip kube-proxy service-HVFWP5L3-ns5/svc5/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 } add chain ip kube-proxy external-HVFWP5L3-ns5/svc5/tcp/p80 @@ -3893,14 +3889,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -3946,21 +3940,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -3969,7 +3960,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element each to cluster-ips and service-ips - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush endpoint chain, add 2 rules ) @@ -3983,7 +3974,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -3993,7 +3983,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4015,14 +4004,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4053,14 +4040,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4096,21 +4081,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4119,7 +4101,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush endpoint chain, add 2 rules ) @@ -4139,14 +4121,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 @@ -4154,7 +4134,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4162,7 +4141,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush endpoint chain, add 2 rules 1, // flush old endpoint chain ) @@ -4185,14 +4164,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4202,7 +4179,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4211,7 +4187,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 4, // add+flush original endpoint chain, add 2 rules 4, // add+flush new endpoint chain, add 2 rules ) @@ -4230,7 +4206,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4241,7 +4216,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4270,14 +4244,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 ip daddr 172.30.0.43 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4287,7 +4259,6 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 ip daddr 172.30.0.44 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4296,7 +4267,7 @@ func TestSyncProxyRulesRepeated(t *testing.T) { assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips - 4, // add+flush service chain, add 2 rules + 3, // add+flush service chain, add 1 rule 8, // add+flush 2 endpoint chains, add 2 rules to each ) @@ -4337,12 +4308,10 @@ func TestSyncProxyRulesStartup(t *testing.T) { add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 @@ -4427,7 +4396,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy no-endpoint-services { 172.30.0.43 . tcp . 80 comment "ns3/svc3:p80" : goto reject-chain } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4437,7 +4405,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 ip daddr 172.30.0.42 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" @@ -4447,7 +4414,7 @@ func TestSyncProxyRulesStartup(t *testing.T) { assertNumOperations(t, nft, setupOps, // nft setup 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set - 8, // add+flush 2 service chains + 2 rules each + 6, // add+flush 2 service chains + 1 rule each 4, // add+flush svc1 endpoint chain + 2 rules ) } @@ -4905,7 +4872,6 @@ func TestBadIPs(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 ip daddr 172.30.0.41 ip saddr != 10.0.0.0/8 mark set mark or 0x4000 comment "masquerade traffic from outside cluster" add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } add chain ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 From 44628d1b6d9905e75bb0829a9d6696dccb922160 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 10:38:51 -0500 Subject: [PATCH 3/4] Do nftables hairpin handling centrally rather than per-endpoint --- pkg/proxy/nftables/helpers_test.go | 26 +++++++ pkg/proxy/nftables/proxier.go | 43 +++++++++--- pkg/proxy/nftables/proxier_test.go | 105 ++++++++++++++++------------- 3 files changed, 115 insertions(+), 59 deletions(-) diff --git a/pkg/proxy/nftables/helpers_test.go b/pkg/proxy/nftables/helpers_test.go index 274af26a569..d2a9c4859b0 100644 --- a/pkg/proxy/nftables/helpers_test.go +++ b/pkg/proxy/nftables/helpers_test.go @@ -162,6 +162,7 @@ type nftablesTracer struct { // additional info about rules we've hit markMasq bool masquerade bool + dnat bool } // newNFTablesTracer creates an nftablesTracer. nodeIPs are the IP to treat as local node @@ -275,6 +276,8 @@ var endpointVMapEntryRegexp = regexp.MustCompile(`\d+ : goto (\S+)`) var masqMarkRegexp = regexp.MustCompile(`^mark set mark or 0x[[:xdigit:]]+$`) var masqCheckRegexp = regexp.MustCompile(`^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+`) var masqueradeRegexp = regexp.MustCompile(`^masquerade fully-random$`) +var dnatCheckRegexp = regexp.MustCompile(`^ct status dnat`) +var hairpinCheckRegexp = regexp.MustCompile(`^ip6* saddr \. ip6* daddr \. meta l4proto \. th dport @hairpin-connections`) var jumpRegexp = regexp.MustCompile(`^(jump|goto) (\S+)$`) var returnRegexp = regexp.MustCompile(`^return$`) var verdictRegexp = regexp.MustCompile(`^(drop|reject)$`) @@ -310,6 +313,28 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP // some of the regexes might match parts of others. switch { + case dnatCheckRegexp.MatchString(rule): + // `^ct status dnat` + // Part of the hairpin check + match := dnatCheckRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if !tracer.dnat { + rule = "" + break + } + + case hairpinCheckRegexp.MatchString(rule): + // `^ip6* saddr \. ip6* daddr \. meta l4proto \. th dport @hairpin-connections` + // Tests whether the packet is hairpinning back to its client. + match := hairpinCheckRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if sourceIP != destIP { + rule = "" + break + } + // HACK: we don't actually bother doing the full lookup; we + // assume that if src and dst matched, it's hairpin. + case destIPOnlyLookupRegexp.MatchString(rule): // `^ip6* daddr @(\S+)` // Tests whether destIP is a member of the indicated set. @@ -552,6 +577,7 @@ func tracePacket(t *testing.T, nft *knftables.Fake, sourceIP, protocol, destIP, if err != nil { t.Errorf("failed to parse host port '%s': %s", tracer.outputs[0], err.Error()) } + tracer.dnat = true } // Run filter-forward, return if packet is terminated. diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index 967c2849e41..7a57e777862 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -93,7 +93,8 @@ const ( firewallCheckChain = "firewall-check" // masquerading - masqueradingChain = "masquerading" + masqueradingChain = "masquerading" + hairpinConnectionsSet = "hairpin-connections" ) // NewDualStackProxier creates a MetaProxier instance, with IPv4 and IPv6 proxies. @@ -197,6 +198,7 @@ type Proxier struct { noEndpointServices *nftElementStorage noEndpointNodePorts *nftElementStorage serviceNodePorts *nftElementStorage + hairpinConnections *nftElementStorage } // Proxier implements proxy.Provider @@ -267,6 +269,7 @@ func NewProxier(ctx context.Context, noEndpointServices: newNFTElementStorage("map", noEndpointServicesMap), noEndpointNodePorts: newNFTElementStorage("map", noEndpointNodePortsMap), serviceNodePorts: newNFTElementStorage("map", serviceNodePortsMap), + hairpinConnections: newNFTElementStorage("set", hairpinConnectionsSet), } logger.V(2).Info("NFTables sync params", "minSyncPeriod", minSyncPeriod, "syncPeriod", syncPeriod, "maxSyncPeriod", proxyutil.FullSyncPeriod) @@ -463,6 +466,13 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { ensureChain(chain, tx, createdChains, false) } + // add hairpin-connections set + tx.Add(&knftables.Set{ + Name: hairpinConnectionsSet, + Type: ipvX_addr + " . " + ipvX_addr + " . inet_proto . inet_service", + Comment: ptr.To("service hairpin connections"), + }) + // Add the rules in the masquerading chain tx.Add(&knftables.Rule{ Chain: masqueradingChain, @@ -472,6 +482,14 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { "masquerade fully-random", ), }) + tx.Add(&knftables.Rule{ + Chain: masqueradingChain, + Rule: knftables.Concat( + "ct status dnat", + ipX, "saddr", ".", ipX, "daddr", ".", "meta l4proto", ".", "th dport", "@", hairpinConnectionsSet, + "masquerade fully-random", + ), + }) // add cluster-ips set. tx.Add(&knftables.Set{ @@ -676,6 +694,7 @@ func (proxier *Proxier) setupNFTables(tx *knftables.Transaction) { proxier.noEndpointServices.readOrReset(tx, proxier.nftables, proxier.logger) proxier.noEndpointNodePorts.readOrReset(tx, proxier.nftables, proxier.logger) proxier.serviceNodePorts.readOrReset(tx, proxier.nftables, proxier.logger) + proxier.hairpinConnections.readOrReset(tx, proxier.nftables, proxier.logger) } // Sync is called to synchronize the proxier state to nftables as soon as possible. @@ -1205,6 +1224,17 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { activeAffinitySets.Insert(epInfo.affinitySetName) } + + // Add endpoint to the hairpin set + proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ + Set: hairpinConnectionsSet, + Key: []string{ + epInfo.IP(), + epInfo.IP(), + protocol, + strconv.Itoa(epInfo.Port()), + }, + }) } } @@ -1620,16 +1650,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { } endpointChain := epInfo.chainName - // Handle traffic that loops back to the originator with SNAT. - tx.Add(&knftables.Rule{ - Chain: endpointChain, - Rule: knftables.Concat( - ipX, "saddr", epInfo.IP(), - proxier.masqueradeRule, - ), - Comment: ptr.To("masquerade hairpin traffic"), - }) - // Handle session affinity if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { tx.Add(&knftables.Rule{ @@ -1686,6 +1706,7 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { proxier.noEndpointServices.cleanupLeftoverKeys(tx) proxier.noEndpointNodePorts.cleanupLeftoverKeys(tx) proxier.serviceNodePorts.cleanupLeftoverKeys(tx) + proxier.hairpinConnections.cleanupLeftoverKeys(tx) // Sync rules. proxier.logger.V(2).Info("Reloading service nftables data", diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index ad991815406..a473ee704a6 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -136,6 +136,7 @@ func NewFakeProxier(ipFamily v1.IPFamily) (*knftables.Fake, *Proxier) { noEndpointServices: newNFTElementStorage("map", noEndpointServicesMap), noEndpointNodePorts: newNFTElementStorage("map", noEndpointNodePortsMap), serviceNodePorts: newNFTElementStorage("map", serviceNodePortsMap), + hairpinConnections: newNFTElementStorage("set", hairpinConnectionsSet), } p.setInitialized(true) p.syncRunner = runner.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, 30*time.Second, time.Minute) @@ -148,6 +149,7 @@ var baseRules = dedent.Dedent(` add set ip kube-proxy cluster-ips { type ipv4_addr ; comment "Active ClusterIPs" ; } add set ip kube-proxy nodeport-ips { type ipv4_addr ; comment "IPs that accept NodePort traffic" ; } + add set ip kube-proxy hairpin-connections { type ipv4_addr . ipv4_addr . inet_proto . inet_service ; comment "service hairpin connections" ; } add map ip kube-proxy firewall-ips { type ipv4_addr . inet_proto . inet_service : verdict ; comment "destinations that are subject to LoadBalancerSourceRanges" ; } add map ip kube-proxy no-endpoint-nodeports { type inet_proto . inet_service : verdict ; comment "vmap to drop or reject packets to service nodeports with no endpoints" ; } @@ -183,6 +185,7 @@ var baseRules = dedent.Dedent(` add rule ip kube-proxy filter-output ct state new jump cluster-ips-check add rule ip kube-proxy firewall-check ip daddr . meta l4proto . th dport vmap @firewall-ips add rule ip kube-proxy masquerading mark and 0x4000 != 0 mark set mark xor 0x4000 masquerade fully-random + add rule ip kube-proxy masquerading ct status dnat ip saddr . ip daddr . meta l4proto . th dport @hairpin-connections masquerade fully-random add rule ip kube-proxy nat-output jump services add rule ip kube-proxy nat-postrouting jump masquerading add rule ip kube-proxy nat-prerouting jump services @@ -366,11 +369,11 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 ip saddr 10.180.0.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.1 . 10.180.0.1 . tcp . 80 } # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 @@ -380,7 +383,6 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local mark set mark or 0x4000 comment "masquerade local traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit local traffic" add chain ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 - add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 ip saddr 10.180.0.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 meta l4proto tcp dnat to 10.180.0.2:80 add element ip kube-proxy cluster-ips { 172.30.0.42 } @@ -388,6 +390,7 @@ func TestOverallNFTablesRules(t *testing.T) { add element ip kube-proxy service-ips { 192.168.99.22 . tcp . 80 : goto external-42NFTM6N-ns2/svc2/tcp/p80 } add element ip kube-proxy service-ips { 1.2.3.4 . tcp . 80 : goto external-42NFTM6N-ns2/svc2/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3001 : goto external-42NFTM6N-ns2/svc2/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.2 . 10.180.0.2 . tcp . 80 } add element ip kube-proxy no-endpoint-nodeports { tcp . 3001 comment "ns2/svc2:p80" : drop } add element ip kube-proxy no-endpoint-services { 1.2.3.4 . tcp . 80 comment "ns2/svc2:p80" : drop } @@ -400,12 +403,12 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 goto service-4AT6LBPK-ns3/svc3/tcp/p80 add chain ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 ip saddr 10.180.0.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3003 : goto external-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.3 . 10.180.0.3 . tcp . 80 } # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 @@ -414,15 +417,15 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 goto service-LAUZTJTB-ns4/svc4/tcp/p80 add chain ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 - add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 ip saddr 10.180.0.5 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 meta l4proto tcp dnat to 10.180.0.5:80 add chain ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 - add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 ip saddr 10.180.0.4 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 meta l4proto tcp dnat to 10.180.0.4:80 add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } add element ip kube-proxy service-ips { 192.168.99.33 . tcp . 80 : goto external-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.4 . 10.180.0.4 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.180.0.5 . 10.180.0.5 . tcp . 80 } # svc5 add set ip kube-proxy affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 { type ipv4_addr ; flags dynamic,timeout ; timeout 10800s ; } @@ -434,7 +437,6 @@ func TestOverallNFTablesRules(t *testing.T) { add rule ip kube-proxy external-HVFWP5L3-ns5/svc5/tcp/p80 goto service-HVFWP5L3-ns5/svc5/tcp/p80 add chain ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 ip saddr 10.180.0.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 update @affinity-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 { ip saddr } add rule ip kube-proxy endpoint-GTK6MW7G-ns5/svc5/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 @@ -446,6 +448,7 @@ func TestOverallNFTablesRules(t *testing.T) { add element ip kube-proxy service-ips { 5.6.7.8 . tcp . 80 : goto external-HVFWP5L3-ns5/svc5/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3002 : goto external-HVFWP5L3-ns5/svc5/tcp/p80 } add element ip kube-proxy firewall-ips { 5.6.7.8 . tcp . 80 : goto firewall-HVFWP5L3-ns5/svc5/tcp/p80 } + # 10.180.0.3:80 is already in hairpin-connections, so we don't want another "add element" line... # svc6 add element ip kube-proxy cluster-ips { 172.30.0.46 } @@ -3887,17 +3890,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -3938,30 +3941,30 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, - 2, // add 1 element each to cluster-ips and service-ips + 3, // add 1 element each to cluster-ips, service-ips, and hairpin-connections 3, // add+flush service chain, add 1 rule - 4, // add+flush endpoint chain, add 2 rules + 3, // add+flush endpoint chain, add 1 rule ) // Delete a service; its chains will be flushed, but not immediately deleted. @@ -3972,11 +3975,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 @@ -3985,12 +3989,11 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, - 2, // delete 1 element each from cluster-ips and service-ips + 3, // delete 1 element each from cluster-ips, service-ips, hairpin-connections 2, // flush 2 chains for service and endpoint ) @@ -4002,17 +4005,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -4038,17 +4041,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } @@ -4079,30 +4082,31 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 ip saddr 10.0.3.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services + 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 4, // add+flush endpoint chain, add 2 rules + 3, // add+flush endpoint chain, add 1 rule ) // Change an endpoint of an existing service. @@ -4119,30 +4123,31 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.2 . 10.0.3.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, + 2, // remove 1 old element from hairpin-connections, add 1 new one 3, // add+flush service chain, add 1 rule - 4, // add+flush endpoint chain, add 2 rules + 3, // add+flush endpoint chain, add 1 rule 1, // flush old endpoint chain ) @@ -4162,34 +4167,35 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.2 . 10.0.3.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.3 . 10.0.3.3 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 ip saddr 10.0.3.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, + 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 4, // add+flush original endpoint chain, add 2 rules - 4, // add+flush new endpoint chain, add 2 rules + 3, // add+flush original endpoint chain, add 1 rule + 3, // add+flush new endpoint chain, add 1 rule ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. @@ -4204,11 +4210,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy no-endpoint-services { 172.30.0.43 . tcp . 80 comment "ns3/svc3:p80" : goto reject-chain } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 @@ -4218,12 +4225,12 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services + 2, // remove 2 elements from hairpin-connections 3, // flush 3 chains ) @@ -4242,33 +4249,34 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.2 . 10.0.3.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.3.3 . 10.0.3.3 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 ip saddr 10.0.3.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 ip saddr 10.0.3.3 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 ip saddr 10.0.4.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips + 2, // add 2 elements to hairpin-connections 3, // add+flush service chain, add 1 rule - 8, // add+flush 2 endpoint chains, add 2 rules to each + 6, // add+flush 2 endpoint chains, add 1 rule to each ) if len(fp.staleChains) != 0 { @@ -4309,17 +4317,17 @@ func TestSyncProxyRulesStartup(t *testing.T) { add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } `)) if err != nil { @@ -4394,28 +4402,29 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 8080 : goto service-MHHHYRWA-ns2/svc2/tcp/p8080 } add element ip kube-proxy no-endpoint-services { 172.30.0.43 . tcp . 80 comment "ns3/svc3:p80" : goto reject-chain } + add element ip kube-proxy hairpin-connections { 10.0.1.1 . 10.0.1.1 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.1.2 . 10.0.1.2 . tcp . 80 } + add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 ip saddr 10.0.1.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 add chain ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 - add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 ip saddr 10.0.1.2 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 ip saddr 10.0.2.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, setupOps, // nft setup + 1, // add new svc1 endpoint to hairpin-connections 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set 6, // add+flush 2 service chains + 1 rule each - 4, // add+flush svc1 endpoint chain + 2 rules + 3, // add+flush svc1 endpoint chain + 1 rule ) } @@ -4879,7 +4888,6 @@ func TestBadIPs(t *testing.T) { add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 goto service-ULMVA6XW-ns1/svc1/tcp/p80 add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 ip saddr 10.180.0.1 mark set mark or 0x4000 comment "masquerade hairpin traffic" add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add chain ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 @@ -4891,6 +4899,7 @@ func TestBadIPs(t *testing.T) { add element ip kube-proxy service-ips { 1.2.3.4 . tcp . 80 : goto external-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy service-nodeports { tcp . 3001 : goto external-ULMVA6XW-ns1/svc1/tcp/p80 } add element ip kube-proxy firewall-ips { 1.2.3.4 . tcp . 80 : goto firewall-ULMVA6XW-ns1/svc1/tcp/p80 } + add element ip kube-proxy hairpin-connections { 10.180.0.1 . 10.180.0.1 . tcp . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) From 782203c96cd0b67d23800fb5fee791e2935d96b3 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 17 Dec 2025 11:14:17 -0500 Subject: [PATCH 4/4] Squash nftables endpoint chains into service vmap We only need a separate chain for the endpoints if the service uses affinity. --- pkg/proxy/nftables/helpers_test.go | 40 +++++- pkg/proxy/nftables/proxier.go | 206 ++++++++++++++--------------- pkg/proxy/nftables/proxier_test.go | 163 ++++++----------------- 3 files changed, 174 insertions(+), 235 deletions(-) diff --git a/pkg/proxy/nftables/helpers_test.go b/pkg/proxy/nftables/helpers_test.go index d2a9c4859b0..daf49b88fd5 100644 --- a/pkg/proxy/nftables/helpers_test.go +++ b/pkg/proxy/nftables/helpers_test.go @@ -270,8 +270,10 @@ var sourceAddrRegexp = regexp.MustCompile(`^ip6* saddr (!= )?(\S+)`) var sourceAddrLookupRegexp = regexp.MustCompile(`^ip6* saddr (!= )?\{([^}]*)\}`) var sourceAddrLocalRegexp = regexp.MustCompile(`^fib saddr type local`) -var endpointVMAPRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) +var endpointVMapRegexp = regexp.MustCompile(`^numgen random mod \d+ vmap \{(.*)\}$`) var endpointVMapEntryRegexp = regexp.MustCompile(`\d+ : goto (\S+)`) +var endpointDNATRegexp = regexp.MustCompile(`^dnat ip6* addr \. port to numgen random mod \d+ map \{(.*)\}$`) +var endpointDNATEntryRegexp = regexp.MustCompile(`\d+ : (\S+) \. (\d+)`) var masqMarkRegexp = regexp.MustCompile(`^mark set mark or 0x[[:xdigit:]]+$`) var masqCheckRegexp = regexp.MustCompile(`^mark and 0x[[:xdigit:]]+ != 0 mark set mark xor 0x[[:xdigit:]]+`) @@ -281,7 +283,8 @@ var hairpinCheckRegexp = regexp.MustCompile(`^ip6* saddr \. ip6* daddr \. meta l var jumpRegexp = regexp.MustCompile(`^(jump|goto) (\S+)$`) var returnRegexp = regexp.MustCompile(`^return$`) var verdictRegexp = regexp.MustCompile(`^(drop|reject)$`) -var dnatRegexp = regexp.MustCompile(`^meta l4proto (tcp|udp|sctp) dnat to (\S+)$`) +var l4protoRegexp = regexp.MustCompile(`^meta l4proto (tcp|udp|sctp)`) +var dnatRegexp = regexp.MustCompile(`^dnat to (\S+)$`) var ignoredRegexp = regexp.MustCompile(strings.Join( []string{ @@ -449,6 +452,15 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP break } + case l4protoRegexp.MatchString(rule): + // `meta l4proto (tcp|udp|sctp)` + match := l4protoRegexp.FindStringSubmatch(rule) + rule = strings.TrimPrefix(rule, match[0]) + if match[1] != protocol { + rule = "" + break + } + case masqMarkRegexp.MatchString(rule): // `^mark set mark or 0x[[:xdigit:]]+$` // Mark for masquerade. @@ -515,20 +527,20 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP return false case dnatRegexp.MatchString(rule): - // `meta l4proto (tcp|udp|sctp) dnat to (\S+)` + // `^dnat to (\S+)$` // DNAT to an endpoint IP and terminate processing. match := dnatRegexp.FindStringSubmatch(rule) - destEndpoint := match[2] + destEndpoint := match[1] tracer.matches = append(tracer.matches, ruleObj.Rule) tracer.outputs = append(tracer.outputs, destEndpoint) return true - case endpointVMAPRegexp.MatchString(rule): + case endpointVMapRegexp.MatchString(rule): // `^numgen random mod \d+ vmap \{(.*)\}$` // Selects a random endpoint and jumps to it. For tracePacket's // purposes, we jump to *all* of the endpoints. - match := endpointVMAPRegexp.FindStringSubmatch(rule) + match := endpointVMapRegexp.FindStringSubmatch(rule) elements := match[1] for _, match = range endpointVMapEntryRegexp.FindAllStringSubmatch(elements, -1) { @@ -543,6 +555,22 @@ func (tracer *nftablesTracer) runChain(chname, sourceIP, protocol, destIP, destP } return true + case endpointDNATRegexp.MatchString(rule): + // `^dnat ip6* addr \. port to numgen random mod \d+ map \{(.*)\}$` + // Selects a random endpoint and DNATs to it. For tracePacket's + // purposes, we DNAT to *all* of the endpoints. + match := endpointDNATRegexp.FindStringSubmatch(rule) + elements := match[1] + + for _, match = range endpointDNATEntryRegexp.FindAllStringSubmatch(elements, -1) { + // `\d+ : (\S+) \. (\d+)` + endpointIP, endpointPort := match[1], match[2] + + tracer.matches = append(tracer.matches, ruleObj.Rule) + tracer.outputs = append(tracer.outputs, net.JoinHostPort(endpointIP, endpointPort)) + } + return true + default: tracer.t.Errorf("unmatched rule: %s", ruleObj.Rule) rule = "" diff --git a/pkg/proxy/nftables/proxier.go b/pkg/proxy/nftables/proxier.go index 7a57e777862..f8bf741984b 100644 --- a/pkg/proxy/nftables/proxier.go +++ b/pkg/proxy/nftables/proxier.go @@ -155,7 +155,6 @@ type Proxier struct { // updating nftables with some partial data after kube-proxy restart. endpointSlicesSynced bool servicesSynced bool - syncedOnce bool lastFullSync time.Time needFullSync bool initialized int32 @@ -1080,7 +1079,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { doFullSync := proxier.needFullSync || (time.Since(proxier.lastFullSync) > proxyutil.FullSyncPeriod) defer func() { - proxier.syncedOnce = true metrics.SyncProxyRulesLatency.WithLabelValues(string(proxier.ipFamily)).Observe(metrics.SinceInSeconds(start)) if !doFullSync { metrics.SyncPartialProxyRulesLatency.WithLabelValues(string(proxier.ipFamily)).Observe(metrics.SinceInSeconds(start)) @@ -1215,27 +1213,33 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { !serviceUpdateResult.UpdatedServices.Has(svcName.NamespacedName) && !endpointUpdateResult.UpdatedServices.Has(svcName.NamespacedName) - // Note the endpoint chains that will be used - for _, ep := range allLocallyReachableEndpoints { - if epInfo, ok := ep.(*endpointInfo); ok { - ensureChain(epInfo.chainName, tx, activeChains, skipServiceUpdate || - proxier.epChainSkipUpdate(existingChains, existingAffinitySets, svcInfo, epInfo)) - // Note the affinity sets that will be used - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - activeAffinitySets.Insert(epInfo.affinitySetName) - } + // We only use separate endpoint chains when adding affinity rules + serviceUsesAffinity := svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP - // Add endpoint to the hairpin set - proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ - Set: hairpinConnectionsSet, - Key: []string{ - epInfo.IP(), - epInfo.IP(), - protocol, - strconv.Itoa(epInfo.Port()), - }, - }) + // Note the endpoints that will be used + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue } + + // If using affinity, note the endpoint chain and affinity set + // names, and ensure that the chain exists. + if serviceUsesAffinity { + ensureChain(epInfo.chainName, tx, activeChains, skipServiceUpdate) + activeAffinitySets.Insert(epInfo.affinitySetName) + } + + // Add endpoint to the hairpin set + proxier.hairpinConnections.ensureElem(tx, &knftables.Element{ + Set: hairpinConnectionsSet, + Key: []string{ + epInfo.IP(), + epInfo.IP(), + protocol, + strconv.Itoa(epInfo.Port()), + }, + }) } // clusterPolicyChain contains the endpoints used with "Cluster" traffic policy @@ -1624,34 +1628,32 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { } } - // If Cluster policy is in use, create the chain and create rules jumping - // from clusterPolicyChain to the clusterEndpoints - if usesClusterPolicyChain { - proxier.writeServiceToEndpointRules(tx, svcInfo, clusterPolicyChain, clusterEndpoints) - } - - // If Local policy is in use, create rules jumping from localPolicyChain - // to the localEndpoints - if usesLocalPolicyChain { - proxier.writeServiceToEndpointRules(tx, svcInfo, localPolicyChain, localEndpoints) - } - - // Generate the per-endpoint chains - for _, ep := range allLocallyReachableEndpoints { - epInfo, ok := ep.(*endpointInfo) - if !ok { - proxier.logger.Error(nil, "Failed to cast endpointInfo", "endpointInfo", ep) - continue + // Write the endpoint rules and/or chains + if !serviceUsesAffinity { + if usesClusterPolicyChain { + proxier.writeServiceToEndpointDNATs(tx, svcInfo, clusterPolicyChain, clusterEndpoints) + } + if usesLocalPolicyChain { + proxier.writeServiceToEndpointDNATs(tx, svcInfo, localPolicyChain, localEndpoints) + } + } else { + if usesClusterPolicyChain { + proxier.writeServiceToEndpointJumps(tx, svcInfo, clusterPolicyChain, clusterEndpoints) + } + if usesLocalPolicyChain { + proxier.writeServiceToEndpointJumps(tx, svcInfo, localPolicyChain, localEndpoints) } - if proxier.epChainSkipUpdate(existingChains, existingAffinitySets, svcInfo, epInfo) { - // If the EP chain is already updated, we can skip it. - continue - } - endpointChain := epInfo.chainName + // And generate the per-endpoint chains and affinity sets + for _, ep := range allLocallyReachableEndpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } - // Handle session affinity - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { + endpointChain := epInfo.chainName + + // Handle session affinity tx.Add(&knftables.Rule{ Chain: endpointChain, Rule: knftables.Concat( @@ -1659,16 +1661,16 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { "{", ipX, "saddr", "}", ), }) - } - // DNAT to final destination. - tx.Add(&knftables.Rule{ - Chain: endpointChain, - Rule: knftables.Concat( - "meta l4proto", protocol, - "dnat to", epInfo.String(), - ), - }) + // DNAT to final destination. + tx.Add(&knftables.Rule{ + Chain: endpointChain, + Rule: knftables.Concat( + "meta l4proto", protocol, + "dnat to", epInfo.String(), + ), + }) + } } } @@ -1765,55 +1767,12 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { return } -// epChainSkipUpdate returns true if the EP chain doesn't need to be updated. -func (proxier *Proxier) epChainSkipUpdate(existingChains, existingAffinitySets sets.Set[string], svcInfo *servicePortInfo, epInfo *endpointInfo) bool { - if proxier.syncedOnce { - // We only skip updating EP chains during the first sync to speed up kube-proxy restart, otherwise return false. - return false - } - if existingChains == nil || existingAffinitySets == nil { - // listing existing objects failed, can't skip updating - return false - } - // EP chain can have up to 3 rules: - // - loopback masquerade rule - // - includes the endpoint IP - // - affinity rule when session affinity is set to ClusterIP - // - includes the affinity set name - // - DNAT rule - // - includes the endpoint IP + port - // EP chain name includes the endpoint IP + port => loopback and DNAT rules are pre-defined by the chain name. - // When session affinity is set to ClusterIP, the affinity set is created for local endpoints. - // Therefore, we can check that sessions affinity hasn't changed by checking if the affinity set exists. - wantAffinitySet := svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP - return existingChains.Has(epInfo.chainName) && wantAffinitySet == existingAffinitySets.Has(epInfo.affinitySetName) -} - -func (proxier *Proxier) writeServiceToEndpointRules(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { - // First write session affinity rules, if applicable. - if svcInfo.SessionAffinityType() == v1.ServiceAffinityClientIP { - ipX := "ip" - if proxier.ipFamily == v1.IPv6Protocol { - ipX = "ip6" - } - - for _, ep := range endpoints { - epInfo, ok := ep.(*endpointInfo) - if !ok { - continue - } - - tx.Add(&knftables.Rule{ - Chain: svcChain, - Rule: knftables.Concat( - ipX, "saddr", "@", epInfo.affinitySetName, - "goto", epInfo.chainName, - ), - }) - } +func (proxier *Proxier) writeServiceToEndpointDNATs(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { + ipX := "ip" + if proxier.ipFamily == v1.IPv6Protocol { + ipX = "ip6" } - // Now write loadbalancing rule var elements []string for i, ep := range endpoints { epInfo, ok := ep.(*endpointInfo) @@ -1822,12 +1781,53 @@ func (proxier *Proxier) writeServiceToEndpointRules(tx *knftables.Transaction, s } elements = append(elements, - strconv.Itoa(i), ":", "goto", epInfo.chainName, + strconv.Itoa(i), ":", epInfo.IP(), ".", strconv.Itoa(epInfo.Port()), ) if i != len(endpoints)-1 { elements = append(elements, ",") } } + tx.Add(&knftables.Rule{ + Chain: svcChain, + Rule: knftables.Concat( + "meta l4proto", strings.ToLower(string(svcInfo.Protocol())), + "dnat", ipX, "addr . port to", + "numgen random mod", len(endpoints), "map", + "{", elements, "}", + ), + }) +} + +func (proxier *Proxier) writeServiceToEndpointJumps(tx *knftables.Transaction, svcInfo *servicePortInfo, svcChain string, endpoints []proxy.Endpoint) { + ipX := "ip" + if proxier.ipFamily == v1.IPv6Protocol { + ipX = "ip6" + } + + var elements []string + // Write the affinity rules, construct the vmap elements + for i, ep := range endpoints { + epInfo, ok := ep.(*endpointInfo) + if !ok { + continue + } + + tx.Add(&knftables.Rule{ + Chain: svcChain, + Rule: knftables.Concat( + ipX, "saddr", "@", epInfo.affinitySetName, + "goto", epInfo.chainName, + ), + }) + + elements = append(elements, + strconv.Itoa(i), ":", "goto", epInfo.chainName, + ) + if i != len(endpoints)-1 { + elements = append(elements, ",") + } + } + // Now write the vmap, for the case where no affinity rule matched tx.Add(&knftables.Rule{ Chain: svcChain, Rule: knftables.Concat( diff --git a/pkg/proxy/nftables/proxier_test.go b/pkg/proxy/nftables/proxier_test.go index a473ee704a6..18fa2cc1140 100644 --- a/pkg/proxy/nftables/proxier_test.go +++ b/pkg/proxy/nftables/proxier_test.go @@ -366,10 +366,7 @@ func TestOverallNFTablesRules(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } - - add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.1 . 80 } add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy service-ips { 172.30.0.41 . tcp . 80 : goto service-ULMVA6XW-ns1/svc1/tcp/p80 } @@ -377,13 +374,11 @@ func TestOverallNFTablesRules(t *testing.T) { # svc2 add chain ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 - add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 } + add rule ip kube-proxy service-42NFTM6N-ns2/svc2/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.2 . 80 } add chain ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 ip saddr 10.0.0.0/8 goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit pod traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local mark set mark or 0x4000 comment "masquerade local traffic" add rule ip kube-proxy external-42NFTM6N-ns2/svc2/tcp/p80 fib saddr type local goto service-42NFTM6N-ns2/svc2/tcp/p80 comment "short-circuit local traffic" - add chain ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 - add rule ip kube-proxy endpoint-SGOXE6O3-ns2/svc2/tcp/p80__10.180.0.2/80 meta l4proto tcp dnat to 10.180.0.2:80 add element ip kube-proxy cluster-ips { 172.30.0.42 } add element ip kube-proxy service-ips { 172.30.0.42 . tcp . 80 : goto service-42NFTM6N-ns2/svc2/tcp/p80 } @@ -398,12 +393,10 @@ func TestOverallNFTablesRules(t *testing.T) { # svc3 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 } + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.3 . 80 } add chain ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-4AT6LBPK-ns3/svc3/tcp/p80 goto service-4AT6LBPK-ns3/svc3/tcp/p80 - add chain ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 - add rule ip kube-proxy endpoint-UEIP74TE-ns3/svc3/tcp/p80__10.180.0.3/80 meta l4proto tcp dnat to 10.180.0.3:80 add element ip kube-proxy cluster-ips { 172.30.0.43 } add element ip kube-proxy service-ips { 172.30.0.43 . tcp . 80 : goto service-4AT6LBPK-ns3/svc3/tcp/p80 } @@ -412,14 +405,10 @@ func TestOverallNFTablesRules(t *testing.T) { # svc4 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 , 1 : goto endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 } + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.180.0.4 . 80 , 1 : 10.180.0.5 . 80 } add chain ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-LAUZTJTB-ns4/svc4/tcp/p80 goto service-LAUZTJTB-ns4/svc4/tcp/p80 - add chain ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 - add rule ip kube-proxy endpoint-5RFCDDV7-ns4/svc4/tcp/p80__10.180.0.5/80 meta l4proto tcp dnat to 10.180.0.5:80 - add chain ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 - add rule ip kube-proxy endpoint-UNZV3OEC-ns4/svc4/tcp/p80__10.180.0.4/80 meta l4proto tcp dnat to 10.180.0.4:80 add element ip kube-proxy cluster-ips { 172.30.0.44 } add element ip kube-proxy service-ips { 172.30.0.44 . tcp . 80 : goto service-LAUZTJTB-ns4/svc4/tcp/p80 } @@ -3894,14 +3883,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) @@ -3946,25 +3931,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 3, // add 1 element each to cluster-ips, service-ips, and hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule ) // Delete a service; its chains will be flushed, but not immediately deleted. @@ -3979,22 +3957,17 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 3, // delete 1 element each from cluster-ips, service-ips, hairpin-connections - 2, // flush 2 chains for service and endpoint + 1, // flush service chain ) // Fake the passage of time and confirm that the stale chains get deleted. @@ -4009,14 +3982,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // delete stale chains happens in a separate transaction, nothing else changed @@ -4045,14 +4014,10 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.3.1 . 10.0.3.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } add element ip kube-proxy no-endpoint-services { 172.30.0.44 . tcp . 80 comment "ns4/svc4:p80" : goto reject-chain } `) @@ -4087,26 +4052,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add rule ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 meta l4proto tcp dnat to 10.0.3.1:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.1 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // add 1 element to service-ips, remove 1 element from no-endpoint-services 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule ) // Change an endpoint of an existing service. @@ -4128,27 +4086,18 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 } - add chain ip kube-proxy endpoint-2OCDJSZQ-ns3/svc3/tcp/p80__10.0.3.1/80 - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.3.2 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 old element from hairpin-connections, add 1 new one 3, // add+flush service chain, add 1 rule - 3, // add+flush endpoint chain, add 1 rule - 1, // flush old endpoint chain ) // (Ensure the old svc3 chain gets deleted in the next sync.) @@ -4173,29 +4122,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.3.2 . 80 , 1 : 10.0.3.3 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) // (The first endpoint chain is unchanged, but the code recreates it anyway.) assertNumOperations(t, nft, 1, // add 1 element to hairpin-connections 3, // add+flush service chain, add 1 rule - 3, // add+flush original endpoint chain, add 1 rule - 3, // add+flush new endpoint chain, add 1 rule ) // Empty a service's endpoints; its chains will be flushed, but not immediately deleted. @@ -4214,27 +4153,21 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from service-ips, add 1 element to no-endpoint-services 2, // remove 2 elements from hairpin-connections - 3, // flush 3 chains + 1, // flush service chain ) - expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80", "endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80", "endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80") + expectedStaleChains := sets.NewString("service-4AT6LBPK-ns3/svc3/tcp/p80") gotStaleChains := sets.StringKeySet(fp.staleChains) if !expectedStaleChains.Equal(gotStaleChains) { t.Errorf("expected stale chains %v, got %v", expectedStaleChains, gotStaleChains) @@ -4255,28 +4188,19 @@ func TestSyncProxyRulesRepeated(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.4.1 . 10.0.4.1 . tcp . 80 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } add chain ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 - add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 , 1 : goto endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 } - add chain ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 - add rule ip kube-proxy endpoint-SWWHDC7X-ns3/svc3/tcp/p80__10.0.3.2/80 meta l4proto tcp dnat to 10.0.3.2:80 - add chain ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 - add rule ip kube-proxy endpoint-TQ2QKHCZ-ns3/svc3/tcp/p80__10.0.3.3/80 meta l4proto tcp dnat to 10.0.3.3:80 + add rule ip kube-proxy service-4AT6LBPK-ns3/svc3/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.3.2 . 80 , 1 : 10.0.3.3 . 80 } add chain ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 - add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 } - add chain ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 - add rule ip kube-proxy endpoint-WAHRBT2B-ns4/svc4/tcp/p80__10.0.4.1/80 meta l4proto tcp dnat to 10.0.4.1:80 + add rule ip kube-proxy service-LAUZTJTB-ns4/svc4/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.4.1 . 80 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, 2, // remove 1 element from no-endpoint-services, add 1 element to service-ips 2, // add 2 elements to hairpin-connections 3, // add+flush service chain, add 1 rule - 6, // add+flush 2 endpoint chains, add 1 rule to each ) if len(fp.staleChains) != 0 { @@ -4312,15 +4236,11 @@ func TestSyncProxyRulesStartup(t *testing.T) { // put a part of desired state to nftables err := nft.ParseDump(baseRules + dedent.Dedent(` add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 } - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.1.1 . 80 } - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } add element ip kube-proxy cluster-ips { 172.30.0.41 } add element ip kube-proxy cluster-ips { 172.30.0.42 } @@ -4407,16 +4327,10 @@ func TestSyncProxyRulesStartup(t *testing.T) { add element ip kube-proxy hairpin-connections { 10.0.2.1 . 10.0.2.1 . tcp . 8080 } add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 2 vmap { 0 : goto endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 , 1 : goto endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 } - add chain ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 - add rule ip kube-proxy endpoint-5TPGNJF2-ns1/svc1/tcp/p80__10.0.1.1/80 meta l4proto tcp dnat to 10.0.1.1:80 - add chain ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 - add rule ip kube-proxy endpoint-ZCZBVNAZ-ns1/svc1/tcp/p80__10.0.1.2/80 meta l4proto tcp dnat to 10.0.1.2:80 + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 2 map { 0 : 10.0.1.1 . 80 , 1 : 10.0.1.2 . 80 } add chain ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 - add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 numgen random mod 1 vmap { 0 : goto endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 } - add chain ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 - add rule ip kube-proxy endpoint-7RVP4LUQ-ns2/svc2/tcp/p8080__10.0.2.1/8080 meta l4proto tcp dnat to 10.0.2.1:8080 + add rule ip kube-proxy service-MHHHYRWA-ns2/svc2/tcp/p8080 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.0.2.1 . 8080 } `) assertNFTablesTransactionEqual(t, getLine(), expected, nft.Dump()) assertNumOperations(t, nft, @@ -4424,7 +4338,6 @@ func TestSyncProxyRulesStartup(t *testing.T) { 1, // add new svc1 endpoint to hairpin-connections 2, // add svc3 IP to the cluster-ips, and to the no-endpoint-services set 6, // add+flush 2 service chains + 1 rule each - 3, // add+flush svc1 endpoint chain + 1 rule ) } @@ -4881,14 +4794,12 @@ func TestBadIPs(t *testing.T) { expected := baseRules + dedent.Dedent(` # svc1 add chain ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 - add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 numgen random mod 1 vmap { 0 : goto endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 } + add rule ip kube-proxy service-ULMVA6XW-ns1/svc1/tcp/p80 meta l4proto tcp dnat ip addr . port to numgen random mod 1 map { 0 : 10.180.0.1 . 80 } add chain ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 mark set mark or 0x4000 comment "masquerade" add rule ip kube-proxy external-ULMVA6XW-ns1/svc1/tcp/p80 goto service-ULMVA6XW-ns1/svc1/tcp/p80 - add chain ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 - add rule ip kube-proxy endpoint-5OJB2KTY-ns1/svc1/tcp/p80__10.180.0.1/80 meta l4proto tcp dnat to 10.180.0.1:80 add chain ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 add rule ip kube-proxy firewall-ULMVA6XW-ns1/svc1/tcp/p80 ip saddr != { 203.0.113.0/25 } drop