From e7a42e8e8e0364561165d2a5e2745addde554c21 Mon Sep 17 00:00:00 2001 From: Praveen Krishna Date: Tue, 28 Oct 2025 17:09:12 +0000 Subject: [PATCH] feat(admission): Add NodeDeclaredFeatures admission plugin --- .../samples/generic/server/admission_test.go | 2 + pkg/kubeapiserver/options/plugins.go | 4 + .../nodedeclaredfeatures/admission.go | 195 +++++++++++++ .../nodedeclaredfeatures/admission_test.go | 275 ++++++++++++++++++ 4 files changed, 476 insertions(+) create mode 100644 plugin/pkg/admission/nodedeclaredfeatures/admission.go create mode 100644 plugin/pkg/admission/nodedeclaredfeatures/admission_test.go diff --git a/pkg/controlplane/apiserver/samples/generic/server/admission_test.go b/pkg/controlplane/apiserver/samples/generic/server/admission_test.go index a16a257840a..b9aab2fd49c 100644 --- a/pkg/controlplane/apiserver/samples/generic/server/admission_test.go +++ b/pkg/controlplane/apiserver/samples/generic/server/admission_test.go @@ -23,6 +23,7 @@ import ( kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" "k8s.io/kubernetes/plugin/pkg/admission/limitranger" "k8s.io/kubernetes/plugin/pkg/admission/network/defaultingressclass" + "k8s.io/kubernetes/plugin/pkg/admission/nodedeclaredfeatures" "k8s.io/kubernetes/plugin/pkg/admission/nodetaint" "k8s.io/kubernetes/plugin/pkg/admission/podtopologylabels" podpriority "k8s.io/kubernetes/plugin/pkg/admission/priority" @@ -44,6 +45,7 @@ var intentionallyOffPlugins = sets.New[string]( defaultingressclass.PluginName, // DefaultIngressClass podsecurity.PluginName, // PodSecurity podtopologylabels.PluginName, // PodTopologyLabels + nodedeclaredfeatures.PluginName, // NodeDeclaredFeatures ) func TestDefaultOffAdmissionPlugins(t *testing.T) { diff --git a/pkg/kubeapiserver/options/plugins.go b/pkg/kubeapiserver/options/plugins.go index d3bf0503c1e..4fb7d0b5b82 100644 --- a/pkg/kubeapiserver/options/plugins.go +++ b/pkg/kubeapiserver/options/plugins.go @@ -44,6 +44,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists" "k8s.io/kubernetes/plugin/pkg/admission/network/defaultingressclass" "k8s.io/kubernetes/plugin/pkg/admission/network/denyserviceexternalips" + "k8s.io/kubernetes/plugin/pkg/admission/nodedeclaredfeatures" "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" "k8s.io/kubernetes/plugin/pkg/admission/nodetaint" "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector" @@ -97,6 +98,7 @@ var AllOrderedPlugins = []string{ defaultingressclass.PluginName, // DefaultIngressClass denyserviceexternalips.PluginName, // DenyServiceExternalIPs podtopologylabels.PluginName, // PodTopologyLabels + nodedeclaredfeatures.PluginName, // NodeDeclaredFeatureValidator // new admission plugins should generally be inserted above here // webhook, resourcequota, and deny plugins must go at the end @@ -149,6 +151,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { ctbattest.Register(plugins) certsubjectrestriction.Register(plugins) podtopologylabels.Register(plugins) + nodedeclaredfeatures.Register(plugins) } // DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver. @@ -176,6 +179,7 @@ func DefaultOffAdmissionPlugins() sets.Set[string] { podtopologylabels.PluginName, // PodTopologyLabels, only active when feature gate PodTopologyLabelsAdmission is enabled. mutatingadmissionpolicy.PluginName, // Mutatingadmissionpolicy, only active when feature gate MutatingAdmissionpolicy is enabled validatingadmissionpolicy.PluginName, // ValidatingAdmissionPolicy, only active when feature gate ValidatingAdmissionPolicy is enabled + nodedeclaredfeatures.PluginName, // NodeDeclaredFeatureValidator, only active when feature gate NodeDeclaredFeatures is enabled ) return sets.New(AllOrderedPlugins...).Difference(defaultOnPlugins) diff --git a/plugin/pkg/admission/nodedeclaredfeatures/admission.go b/plugin/pkg/admission/nodedeclaredfeatures/admission.go new file mode 100644 index 00000000000..a05a67e6f06 --- /dev/null +++ b/plugin/pkg/admission/nodedeclaredfeatures/admission.go @@ -0,0 +1,195 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodedeclaredfeatures + +import ( + "context" + "fmt" + "io" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + versionutil "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/admission" + genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer" + "k8s.io/client-go/informers" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/component-base/featuregate" + "k8s.io/component-base/version" + ndf "k8s.io/component-helpers/nodedeclaredfeatures" + ndffeatures "k8s.io/component-helpers/nodedeclaredfeatures/features" + "k8s.io/kubernetes/pkg/apis/core" + v1 "k8s.io/kubernetes/pkg/apis/core/v1" + "k8s.io/kubernetes/pkg/features" +) + +const ( + // PluginName is the name of this admission controller plugin. + PluginName = "NodeDeclaredFeatureValidator" +) + +// Register registers a plugin. +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + return NewPlugin() + }) +} + +// Plugin is an admission controller that validates pod updates against node capabilities. +type Plugin struct { + *admission.Handler + nodeLister corev1listers.NodeLister + nodeDeclaredFeatureFramework *ndf.Framework + nodeDeclaredFeatureGateEnabled bool + version *versionutil.Version +} + +var _ admission.ValidationInterface = &Plugin{} +var _ = genericadmissioninitializer.WantsExternalKubeInformerFactory(&Plugin{}) +var _ genericadmissioninitializer.WantsFeatures = &Plugin{} + +// New creates a new Plugin admission plugin +func NewPlugin() (*Plugin, error) { + framework, err := ndf.New(ndffeatures.AllFeatures) + if err != nil { + return nil, fmt.Errorf("failed to create node declared features helper: %w", err) + } + + ver, err := versionutil.Parse(version.Get().String()) + if err != nil { + return nil, fmt.Errorf("failed to parse version: %w", err) + } + return &Plugin{ + Handler: admission.NewHandler(admission.Update), + nodeDeclaredFeatureFramework: framework, + version: ver, + }, nil +} + +// SetExternalKubeInformerFactory sets the informer factory for the plugin. +func (p *Plugin) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { + nodeInformer := f.Core().V1().Nodes() + p.nodeLister = f.Core().V1().Nodes().Lister() + p.SetReadyFunc(nodeInformer.Informer().HasSynced) +} + +// SetFeatures sets the feature gates for the plugin. +func (p *Plugin) InspectFeatureGates(featureGates featuregate.FeatureGate) { + p.nodeDeclaredFeatureGateEnabled = featureGates.Enabled(features.NodeDeclaredFeatures) +} + +// ValidateInitialization ensures that the plugin is properly initialized. +func (p *Plugin) ValidateInitialization() error { + if p.nodeLister == nil { + return fmt.Errorf("missing node lister for %s", PluginName) + } + + if p.nodeDeclaredFeatureFramework == nil { + framework, err := ndf.New(ndffeatures.AllFeatures) + if err != nil { + return fmt.Errorf("failed to create node feature helper for %s: %w", PluginName, err) + } + p.nodeDeclaredFeatureFramework = framework + } + return nil +} + +// Validate is the core of the admission controller logic. +func (p *Plugin) Validate(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error { + // Ignore if the feature gate is not enabled. + if !p.nodeDeclaredFeatureGateEnabled { + return nil + } + + if a.GetOperation() != admission.Update { + return nil + } + + // We only care about Pod updates. + if a.GetResource().GroupResource() != core.Resource("pods") { + return nil + } + + // Only validate updates to the main pod spec (subresource == "") + // or the custom "resize" subresource. + subresource := a.GetSubresource() + if subresource != "" && subresource != "resize" { + return nil + } + pod, ok := a.GetObject().(*core.Pod) + if !ok { + return errors.NewBadRequest(fmt.Sprintf("expected a pod but got a %T", a.GetObject())) + } + // We only care about pods that are already bound to a node. + if len(pod.Spec.NodeName) == 0 { + return nil + } + oldPod, ok := a.GetOldObject().(*core.Pod) + if !ok { + return errors.NewBadRequest(fmt.Sprintf("expected an old pod but got a %T", a.GetOldObject())) + } + + if oldPod.Generation == pod.Generation { + // since generation is only incremented when the spec changes, we can skip validation if it doesnt. + return nil + } + + if !p.WaitForReady() { + return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request")) + } + + return p.validatePodUpdate(pod, oldPod, a) +} + +func (p *Plugin) validatePodUpdate(pod, oldPod *core.Pod, a admission.Attributes) error { + // Convert internal to external pods for the helper library. + podV1 := &corev1.Pod{} + if err := v1.Convert_core_Pod_To_v1_Pod(pod, podV1, nil); err != nil { + return errors.NewBadRequest(fmt.Sprintf("failed to convert pod: %v", err)) + } + oldPodV1 := &corev1.Pod{} + if err := v1.Convert_core_Pod_To_v1_Pod(oldPod, oldPodV1, nil); err != nil { + return errors.NewBadRequest(fmt.Sprintf("failed to convert oldPod: %v", err)) + } + oldPodInfo := &ndf.PodInfo{Spec: &oldPodV1.Spec, Status: &oldPodV1.Status} + newPodInfo := &ndf.PodInfo{Spec: &podV1.Spec, Status: &podV1.Status} + reqs, err := p.nodeDeclaredFeatureFramework.InferForPodUpdate(oldPodInfo, newPodInfo, p.version) + if err != nil { + return admission.NewForbidden(a, fmt.Errorf("failed to infer pod capability requirements: %w", err)) + } + // If there are no specific feature requirements for this update, we're done. + if reqs.Len() == 0 { + return nil + } + node, err := p.nodeLister.Get(pod.Spec.NodeName) + if err != nil { + if errors.IsNotFound(err) { + return admission.NewForbidden(a, fmt.Errorf("node %q not found", pod.Spec.NodeName)) + } + return admission.NewForbidden(a, fmt.Errorf("failed to get node %q: %w", pod.Spec.NodeName, err)) + } + result, err := ndf.MatchNode(reqs, node) + if err != nil { + return admission.NewForbidden(a, fmt.Errorf("failed to match pod requirements against node %q: %w", node.Name, err)) + } + if !result.IsMatch { + return admission.NewForbidden(a, fmt.Errorf("pod update requires features %s which are not available on node %q", strings.Join(result.UnsatisfiedRequirements, ", "), node.Name)) + } + + return nil +} diff --git a/plugin/pkg/admission/nodedeclaredfeatures/admission_test.go b/plugin/pkg/admission/nodedeclaredfeatures/admission_test.go new file mode 100644 index 00000000000..be4fe78f8e5 --- /dev/null +++ b/plugin/pkg/admission/nodedeclaredfeatures/admission_test.go @@ -0,0 +1,275 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodedeclaredfeatures + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apiserver/pkg/admission" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + featuregatetesting "k8s.io/component-base/featuregate/testing" + ndf "k8s.io/component-helpers/nodedeclaredfeatures" + ndftesting "k8s.io/component-helpers/nodedeclaredfeatures/testing" + "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" +) + +func TestAdmission(t *testing.T) { + nodeWithFeature := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + Status: v1.NodeStatus{ + DeclaredFeatures: []string{"TestFeature"}, + NodeInfo: v1.NodeSystemInfo{KubeletVersion: "1.35.0"}, + }, + } + nodeWithoutFeature := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node-no-feature"}, + Status: v1.NodeStatus{ + DeclaredFeatures: []string{}, + NodeInfo: v1.NodeSystemInfo{KubeletVersion: "1.35.0"}, + }, + } + + client := fake.NewClientset(nodeWithFeature, nodeWithoutFeature) + informerFactory := informers.NewSharedInformerFactory(client, 0) + + oldPod := &core.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "test-ns"}, + Spec: core.PodSpec{ + NodeName: "test-node", + Containers: []core.Container{ + { + Name: "container", + Image: "image", + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceCPU: resource.MustParse("1000m"), + core.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }}, + }, + } + newPod := oldPod.DeepCopy() + newPod.Generation = oldPod.Generation + 1 + newPod.Spec.Containers[0].Resources.Requests[core.ResourceCPU] = resource.MustParse("2000m") + podWithNoNode := oldPod.DeepCopy() + podWithNoNode.Spec.NodeName = "" + podWithInvalidNode := oldPod.DeepCopy() + podWithInvalidNode.Spec.NodeName = "invalid-node" + + createMockFeature := func(t *testing.T, name string, inferForUpdate bool, maxVersionStr string) *ndftesting.MockFeature { + m := ndftesting.NewMockFeature(t) + m.EXPECT().Name().Return(name).Maybe() + m.EXPECT().InferForUpdate(mock.Anything, mock.Anything).Return(inferForUpdate).Maybe() + if maxVersionStr != "" { + minVersion := version.MustParseSemantic(maxVersionStr) + m.EXPECT().MaxVersion().Return(minVersion).Maybe() + } else { + m.EXPECT().MaxVersion().Return(nil).Maybe() + } + return m + } + + testCases := []struct { + name string + pod *core.Pod + oldPod *core.Pod + registeredFeatures []ndf.Feature + featureGateEnabled bool + componentVersion string + expectErr bool + errContains string + subresource string + }{ + { + name: "Feature gate disabled", + pod: newPod, + oldPod: oldPod, + registeredFeatures: []ndf.Feature{}, + featureGateEnabled: false, + expectErr: false, + componentVersion: "1.35.0", + }, + { + name: "skip validation when pod is not bound to node", + pod: podWithNoNode, + oldPod: oldPod, + registeredFeatures: []ndf.Feature{}, + featureGateEnabled: true, + expectErr: false, + componentVersion: "1.35.0", + }, + { + name: "skip validation on invalid node name", + pod: func() *core.Pod { p := newPod.DeepCopy(); p.Spec.NodeName = "not-found-node"; return p }(), + oldPod: oldPod, + featureGateEnabled: true, + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "TestFeature", true, "1.35.0"), + }, + expectErr: true, + errContains: "node \"not-found-node\" not found", + componentVersion: "1.35.0", + }, + { + name: "No feature requirements", + pod: newPod, + oldPod: oldPod, + featureGateEnabled: true, + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "FeatureA", false, ""), + }, + componentVersion: "1.35.0", + expectErr: false, + }, + { + name: "Feature requirement met", + pod: newPod, + oldPod: oldPod, + featureGateEnabled: true, + componentVersion: "1.35.0", + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "TestFeature", true, "1.35.0"), + }, + expectErr: false, + }, + { + name: "Feature requirement not met", + pod: func() *core.Pod { + p := newPod.DeepCopy() + p.Spec.NodeName = "test-node-no-feature" + return p + }(), + oldPod: oldPod, + featureGateEnabled: true, + componentVersion: "1.34.0", + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "TestFeature", true, "1.35.0"), + }, + expectErr: true, + errContains: "pod update requires features TestFeature which are not available on node \"test-node-no-feature\"", + }, + { + name: "skip validation when generation not updated", + pod: func() *core.Pod { + p := newPod.DeepCopy() + p.Generation = oldPod.Generation + return p + }(), + oldPod: oldPod, + featureGateEnabled: true, + componentVersion: "1.34.0", + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "TestFeature", true, "1.35.0"), + }, + expectErr: false, + }, + { + name: "Feature not need as its generally available", + pod: func() *core.Pod { + p := newPod.DeepCopy() + p.Spec.NodeName = "test-node-no-feature" + return p + }(), + oldPod: oldPod, + featureGateEnabled: true, + componentVersion: "1.35.0", + registeredFeatures: []ndf.Feature{ + // Feature max version less than component version + createMockFeature(t, "TestFeature", false, "1.34.0"), + }, + expectErr: false, + }, + { + name: "skip validation for `status` subresource", + pod: func() *core.Pod { + p := newPod.DeepCopy() + p.Spec.NodeName = "test-node-no-feature" + return p + }(), + oldPod: oldPod, + featureGateEnabled: true, + componentVersion: "1.35.0", + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "TestFeature", true, "1.35.0"), + }, + subresource: "status", + expectErr: false, + }, + { + name: "DO not skip validation for `resize` subresource", + pod: newPod, + oldPod: oldPod, + featureGateEnabled: true, + componentVersion: "1.34.0", + registeredFeatures: []ndf.Feature{ + createMockFeature(t, "TestFeature", true, "1.35.0"), + }, + subresource: "resize", + expectErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeDeclaredFeatures, tc.featureGateEnabled) + + target, err := NewPlugin() + require.NoError(t, err) + + if tc.featureGateEnabled { + framework, err := ndf.New(tc.registeredFeatures) + require.NoError(t, err) + target.nodeDeclaredFeatureFramework = framework + target.version = version.MustParseSemantic(tc.componentVersion) + } + + target.SetExternalKubeInformerFactory(informerFactory) + target.InspectFeatureGates(utilfeature.DefaultFeatureGate) + err = target.ValidateInitialization() + require.NoError(t, err) + + stopCh := make(chan struct{}) + defer close(stopCh) + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + attrs := admission.NewAttributesRecord(tc.pod, tc.oldPod, core.Kind("Pod").WithVersion("v1"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("v1"), tc.subresource, admission.Update, &metav1.UpdateOptions{}, false, nil) + err = target.Validate(context.Background(), attrs, nil) + + if tc.expectErr { + require.Error(t, err) + if tc.errContains != "" { + assert.Contains(t, err.Error(), tc.errContains) + } + } else { + require.NoError(t, err) + } + }) + } +}