From 6f9b630e132109ab7c75d80e2d8f65d3bb0a5820 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Thu, 16 Oct 2025 23:01:31 +0530 Subject: [PATCH 1/3] [kubelet] add new `OnPodSandboxReady` method to RuntimeHelper interface to update `PodReadyToStartContainers` condition immediately after sandbox creation This is to address the bug (gh-issue 134460), which reported that currently `PodReadyToStartContainers` condition is only set to `True` after the container image pull is completed. so, if the image size is big and image pull takes significant time to finish, the pod status managaer is blocked and the condition remaind `False`. The commit implements the following changes, to allow kubelet to update the `PodReadyToStartContainers` pod condition immediately after all three requirements (pod sandbox, networking, volume)are ready, but before container images are pulled or containers are created. * add `OnPodSandboxReady` method to the `RuntimeHelper` interface in `container/helpers.go` * implement the `OnPodSandboxReady` method in Kubelet * inside `(containerRuntime).SyncPod`, after sandbox creation and network configuration, invoke `runtimeHelper.OnPodSandboxReady()` directly (this method retrieves current pod status, generates updated API status, and notifies the status manager to sync to the API server) This implementation is gated under `PodReadyToStartContainersCondition` feature gate, and fails gracefully, i.e, it only logs error and continues the pod creation process to make sure that these new changes don't block pod startup. --- pkg/kubelet/container/helpers.go | 4 ++ pkg/kubelet/kubelet.go | 43 +++++++++++++++++++ .../kuberuntime/kuberuntime_manager.go | 27 ++++++++---- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index 59f470c597c..0e7fff51369 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -75,6 +75,10 @@ type RuntimeHelper interface { // PodCPUAndMemoryStats reads the latest CPU & memory usage stats. PodCPUAndMemoryStats(context.Context, *v1.Pod, *PodStatus) (*statsapi.PodStats, error) + + // OnPodSandboxReady callback is invoked after pod sandbox, networking, volume are ready. + // This is used to update the PodReadyToStartContainers condition. + OnPodSandboxReady(ctx context.Context, pod *v1.Pod) error } // ShouldContainerBeRestarted checks whether a container needs to be restarted. diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index fc83e5449eb..b4503a6b4ce 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -50,6 +50,7 @@ import ( v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" utilfs "k8s.io/kubernetes/pkg/util/filesystem" + utilpod "k8s.io/kubernetes/pkg/util/pod" netutils "k8s.io/utils/net" "k8s.io/utils/ptr" @@ -3381,3 +3382,45 @@ func (kl *Kubelet) fastStaticPodsRegistration(ctx context.Context) { func (kl *Kubelet) SetPodWatchCondition(podUID types.UID, conditionKey string, condition pleg.WatchCondition) { kl.pleg.SetPodWatchCondition(podUID, conditionKey, condition) } + +// OnPodSandboxReady is the callback implementation invoked by the container runtime after +// all three requirements (sandbox, networking, volumes) are ready to immediately update +// the `PodReadyToStartContainers` pod status condition to `True`. +// This method implements the RuntimeHelper interface. +// Ref: https://github.com/kubernetes/kubernetes/issues/134460 +func (kl *Kubelet) OnPodSandboxReady(ctx context.Context, pod *v1.Pod) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.PodReadyToStartContainersCondition) { + return nil + } + + logger := klog.FromContext(ctx) + logger.V(3).Info("OnPodSandboxReady callback invoked", "pod", klog.KObj(pod), "podUID", pod.UID) + + existingStatus, ok := kl.statusManager.GetPodStatus(pod.UID) + if !ok { + existingStatus = pod.Status + } + + readySandboxCondition := v1.PodCondition{ + Type: v1.PodReadyToStartContainers, + Status: v1.ConditionTrue, + ObservedGeneration: podutil.CalculatePodConditionObservedGeneration(&existingStatus, pod.Generation, v1.PodReadyToStartContainers), + } + + lastTransitionTime := metav1.Now() + _, existingCondition := podutil.GetPodCondition(&existingStatus, v1.PodReadyToStartContainers) + if existingCondition != nil && existingCondition.Status == readySandboxCondition.Status { + lastTransitionTime = existingCondition.LastTransitionTime + } + readySandboxCondition.LastTransitionTime = lastTransitionTime + + existingStatus.Conditions = utilpod.ReplaceOrAppendPodCondition(existingStatus.Conditions, &readySandboxCondition) + + kl.statusManager.SetPodStatus(logger, pod, existingStatus) + + logger.V(3).Info("Successfully updated PodReadyToStartContainers condition after sandbox creation", + "pod", klog.KObj(pod), + "podUID", pod.UID) + + return nil +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 46eec1c2abc..7546e16204e 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -1387,10 +1387,11 @@ func (m *kubeGenericRuntimeManager) computePodLevelResourcesResizeAction(ctx con // 2. Kill pod sandbox if necessary. // 3. Kill any containers that should not be running. // 4. Create sandbox if necessary. -// 5. Create ephemeral containers. -// 6. Create init containers. -// 7. Resize running containers (if InPlacePodVerticalScaling==true) -// 8. Create normal containers. +// 5. Invoke OnPodSandboxReady to notify Kubelet to update pod status. +// 6. Create ephemeral containers. +// 7. Create init containers. +// 8. Resize running containers (if InPlacePodVerticalScaling==true) +// 9. Create normal containers. func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff, restartAllContainers bool) (result kubecontainer.PodSyncResult) { logger := klog.FromContext(ctx) // Step 1: Compute sandbox and container changes. @@ -1588,6 +1589,16 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po podIPs = m.determinePodSandboxIPs(ctx, pod.Namespace, pod.Name, resp.GetStatus()) logger.V(4).Info("Determined the ip for pod after sandbox changed", "IPs", podIPs, "pod", klog.KObj(pod)) } + + // Step 5: invoke the sandbox ready callback before image pulling . + // At this point, dynamic resources are prepared (at PrepareDynamicResources() call above) + // and volumes are already mounted (at the kubelet SyncPod() level), so, + // all requirements (sandbox, networking, volumes) are met to set `PodReadyToStartContainers=True` condition. + logger.V(4).Info("Pod sandbox and network ready, invoking callback", "pod", klog.KObj(pod), "podIPs", podIPs) + if err := m.runtimeHelper.OnPodSandboxReady(ctx, pod); err != nil { + // log the error but continue the pod creation process, to retain the existing behaviour + logger.Error(err, "Failed to invoke sandbox ready callback, continuing with pod creation", "pod", klog.KObj(pod)) + } } // the start containers routines depend on pod ip(as in primary pod ip) @@ -1669,7 +1680,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po return nil } - // Step 5: start ephemeral containers + // Step 6: start ephemeral containers // These are started "prior" to init containers to allow running ephemeral containers even when there // are errors starting an init container. In practice init containers will start first since ephemeral // containers cannot be specified on pod creation. @@ -1677,7 +1688,7 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po start(ctx, "ephemeral container", metrics.EphemeralContainer, ephemeralContainerStartSpec(&pod.Spec.EphemeralContainers[idx])) } - // Step 6: start init containers. + // Step 7: start init containers. for _, idx := range podContainerChanges.InitContainersToStart { container := &pod.Spec.InitContainers[idx] // Start the next init container. @@ -1694,14 +1705,14 @@ func (m *kubeGenericRuntimeManager) SyncPod(ctx context.Context, pod *v1.Pod, po logger.V(4).Info("Completed init container for pod", "containerName", container.Name, "pod", klog.KObj(pod)) } - // Step 7: For containers in podContainerChanges.ContainersToUpdate[CPU,Memory] list, invoke UpdateContainerResources + // Step 8: For containers in podContainerChanges.ContainersToUpdate[CPU,Memory] list, invoke UpdateContainerResources if resizable, _, _ := allocation.IsInPlacePodVerticalScalingAllowed(pod); resizable { if len(podContainerChanges.ContainersToUpdate) > 0 || podContainerChanges.UpdatePodResources || podContainerChanges.UpdatePodLevelResources { result.SyncResults = append(result.SyncResults, m.doPodResizeAction(ctx, pod, podStatus, podContainerChanges)) } } - // Step 8: start containers in podContainerChanges.ContainersToStart. + // Step 9: start containers in podContainerChanges.ContainersToStart. for _, idx := range podContainerChanges.ContainersToStart { start(ctx, "container", metrics.Container, containerStartSpec(&pod.Spec.Containers[idx])) } From 4a434ebab7baaa49e3ccd729e3a6bd4630d403f2 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Thu, 16 Oct 2025 23:23:38 +0530 Subject: [PATCH 2/3] add tests to verify invocation of OnPodSandboxReady method and PodReadyToStartContainers condition --- .../container/testing/fake_runtime_helper.go | 5 + .../kuberuntime/kuberuntime_manager_test.go | 284 ++++++++++++++++++ 2 files changed, 289 insertions(+) diff --git a/pkg/kubelet/container/testing/fake_runtime_helper.go b/pkg/kubelet/container/testing/fake_runtime_helper.go index d7da9e33b2f..67c064e8e2d 100644 --- a/pkg/kubelet/container/testing/fake_runtime_helper.go +++ b/pkg/kubelet/container/testing/fake_runtime_helper.go @@ -128,3 +128,8 @@ func (f *FakeRuntimeHelper) PodCPUAndMemoryStats(_ context.Context, pod *v1.Pod, } return nil, fmt.Errorf("stats for pod %q not found", pod.UID) } + +func (f *FakeRuntimeHelper) OnPodSandboxReady(_ context.Context, _ *v1.Pod) error { + // Not implemented + return nil +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index b27dcdc128b..93443de614a 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -5121,3 +5121,287 @@ func TestCmpActuatedAllocated(t *testing.T) { }) } } + +// testRuntimeHelper implements the RuntimeHelper interface for testing OnPodSandboxReady invocation. +type testRuntimeHelper struct { + *containertest.FakeRuntimeHelper + onPodSandboxReadyCalled bool + onPodSandboxReadyPod *v1.Pod + onPodSandboxReadyCtx context.Context + onPodSandboxReadyError error + captureStateFunc func() // optional function to capture state when OnPodSandboxReady is called + prepareDynamicResourcesCalled bool + prepareDynamicResourcesError error +} + +func (t *testRuntimeHelper) OnPodSandboxReady(ctx context.Context, pod *v1.Pod) error { + t.onPodSandboxReadyCalled = true + t.onPodSandboxReadyPod = pod + t.onPodSandboxReadyCtx = ctx + if t.captureStateFunc != nil { + t.captureStateFunc() + } + return t.onPodSandboxReadyError +} + +func (t *testRuntimeHelper) PrepareDynamicResources(ctx context.Context, pod *v1.Pod) error { + t.prepareDynamicResourcesCalled = true + return t.prepareDynamicResourcesError +} + +// TestOnPodSandboxReadyInvocation verifies OnPodSandboxReady is called at the correct time +// and validates the order between the DRA allocate calls and PodReadytoStartContainers condition. +// It works in the following order: +// 1. setup test helper and inject errors +// 2. create pod (with/without devices) +// 3. run SyncPod +// 4. verify device allocation +// 5. verify OnPodSandboxReady invocation +// 6. verify final pod state +func TestOnPodSandboxReadyInvocation(t *testing.T) { + tCtx := ktesting.Init(t) + + tests := []struct { + name string + onPodSandboxReadyShouldErr bool + deviceAllocationShouldErr bool + expectOnPodSandboxReady bool + expectSyncPodSuccess bool + expectDeviceAllocation bool + enablePodReadyToStartContainers bool + description string + }{ + { + name: "OnPodSandboxReady succeeds with feature enabled", + onPodSandboxReadyShouldErr: false, + deviceAllocationShouldErr: false, + expectOnPodSandboxReady: true, + expectSyncPodSuccess: true, + expectDeviceAllocation: false, + enablePodReadyToStartContainers: true, + description: "Verifies OnPodSandboxReady is called and succeeds with PodReadyToStartContainersCondition feature gate enabled", + }, + { + name: "OnPodSandboxReady succeeds with feature disabled", + onPodSandboxReadyShouldErr: false, + deviceAllocationShouldErr: false, + expectOnPodSandboxReady: true, + expectSyncPodSuccess: true, + expectDeviceAllocation: false, + enablePodReadyToStartContainers: false, + description: "Verifies OnPodSandboxReady is called and succeeds with PodReadyToStartContainersCondition feature gate disabled", + }, + { + name: "OnPodSandboxReady fails but SyncPod continues with feature enabled", + onPodSandboxReadyShouldErr: true, + deviceAllocationShouldErr: false, + expectOnPodSandboxReady: true, + expectSyncPodSuccess: true, // SyncPod still succeed even if OnPodSandboxReady fails + expectDeviceAllocation: false, + enablePodReadyToStartContainers: true, + description: "Verifies OnPodSandboxReady errors don't block pod creation with PodReadyToStartContainersCondition feature gate enabled", + }, + { + name: "OnPodSandboxReady fails but SyncPod continues with feature disabled", + onPodSandboxReadyShouldErr: true, + deviceAllocationShouldErr: false, + expectOnPodSandboxReady: true, + expectSyncPodSuccess: true, // SyncPod still succeed even if OnPodSandboxReady fails + expectDeviceAllocation: false, + enablePodReadyToStartContainers: false, + description: "Verifies OnPodSandboxReady errors don't block pod creation with PodReadyToStartContainersCondition feature gate disabled", + }, + { + name: "PrepareDynamicResources (device allocation) called before OnPodSandboxReady with feature enabled", + onPodSandboxReadyShouldErr: false, + deviceAllocationShouldErr: false, + expectOnPodSandboxReady: true, + expectSyncPodSuccess: true, + expectDeviceAllocation: true, + enablePodReadyToStartContainers: true, + description: "Verifies the order (PrepareDynamicResources -> OnPodSandboxReady) in case of pod with ResourceClaims with PodReadyToStartContainersCondition feature gate enabled", + }, + { + name: "PrepareDynamicResources (device allocation) called before OnPodSandboxReady with feature disabled", + onPodSandboxReadyShouldErr: false, + deviceAllocationShouldErr: false, + expectOnPodSandboxReady: true, + expectSyncPodSuccess: true, + expectDeviceAllocation: true, + enablePodReadyToStartContainers: false, + description: "Verifies the order (PrepareDynamicResources -> OnPodSandboxReady) in case of pod with ResourceClaims with PodReadyToStartContainersCondition feature gate disabled", + }, + { + name: "PrepareDynamicResources (device allocation) failure prevents sandbox creation with feature enabled", + onPodSandboxReadyShouldErr: false, + deviceAllocationShouldErr: true, + expectOnPodSandboxReady: false, + expectSyncPodSuccess: true, // SyncPod doesn't return error, just returns early if `PrepareDynamicResources` call ends up failing + expectDeviceAllocation: true, + enablePodReadyToStartContainers: true, + description: "Verifies PrepareDynamicResources failure causes early return in case of pod with ResourceClaims with PodReadyToStartContainersCondition feature gate enabled", + }, + { + name: "PrepareDynamicResources (device allocation) failure prevents sandbox creation with feature disabled", + onPodSandboxReadyShouldErr: false, + deviceAllocationShouldErr: true, + expectOnPodSandboxReady: false, + expectSyncPodSuccess: true, // SyncPod doesn't return error, just returns early if `PrepareDynamicResources` call ends up failing + expectDeviceAllocation: true, + enablePodReadyToStartContainers: false, + description: "Verifies PrepareDynamicResources failure causes early return in case of pod with ResourceClaims with PodReadyToStartContainersCondition feature gate disabled", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodReadyToStartContainersCondition, test.enablePodReadyToStartContainers) + + // step 1 - setup test helper and inject errors + fakeRuntime, fakeImage, m, err := createTestRuntimeManager(tCtx) + require.NoError(t, err) + + testHelper := &testRuntimeHelper{ + FakeRuntimeHelper: &containertest.FakeRuntimeHelper{}, + } + if test.onPodSandboxReadyShouldErr { + testHelper.onPodSandboxReadyError = fmt.Errorf("OnPodSandboxReady intentionally failed for testing") + } + if test.deviceAllocationShouldErr { + testHelper.prepareDynamicResourcesError = fmt.Errorf("PrepareDynamicResources intentionally failed for testing") + } + m.runtimeHelper = testHelper + + // step 2 - create pod (with/without devices) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "test-pod-uid", + Name: "test-pod", + Namespace: "test-namespace", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + }, + } + if test.expectDeviceAllocation { + pod.Spec.ResourceClaims = []v1.PodResourceClaim{ + { + Name: "test-device", + }, + } + } + + // step 3 - run SyncPod + backOff := flowcontrol.NewBackOff(time.Second, time.Minute) + result := m.SyncPod(tCtx, pod, &kubecontainer.PodStatus{}, []v1.Secret{}, backOff, false) + + if test.expectSyncPodSuccess { + require.NoError(t, result.Error(), test.description) + } else { + require.Error(t, result.Error(), test.description) + } + + // step 4 - verify device allocation + if test.expectDeviceAllocation { + require.True(t, testHelper.prepareDynamicResourcesCalled, + "PrepareDynamicResources should be called for pods with resource claims") + + if test.expectOnPodSandboxReady && testHelper.onPodSandboxReadyCalled { + require.True(t, testHelper.prepareDynamicResourcesCalled, + "PrepareDynamicResources must be called before OnPodSandboxReady") + } + } + + // step 5 - verify OnPodSandboxReady invocation + assert.Equal(t, test.expectOnPodSandboxReady, testHelper.onPodSandboxReadyCalled, "OnPodSandboxReady invocation mismatch: "+test.description) + + if test.expectOnPodSandboxReady { + assert.NotNil(t, testHelper.onPodSandboxReadyPod, "OnPodSandboxReady should receive pod") + assert.Equal(t, pod.UID, testHelper.onPodSandboxReadyPod.UID, "OnPodSandboxReady should receive correct pod UID") + assert.Equal(t, pod.Name, testHelper.onPodSandboxReadyPod.Name, "OnPodSandboxReady should receive correct pod name") + assert.Equal(t, pod.Namespace, testHelper.onPodSandboxReadyPod.Namespace, "OnPodSandboxReady should receive correct pod namespace") + assert.NotNil(t, testHelper.onPodSandboxReadyCtx, "OnPodSandboxReady should receive context") + + require.Len(t, fakeRuntime.Sandboxes, 1, "sandbox should be created before OnPodSandboxReady") + for _, sandbox := range fakeRuntime.Sandboxes { + require.Equal(t, runtimeapi.PodSandboxState_SANDBOX_READY, sandbox.State, "sandbox should be ready when OnPodSandboxReady is invoked") + } + } + + // step 6 - verify the final pod state + if test.expectSyncPodSuccess && !test.deviceAllocationShouldErr { + assert.Len(t, fakeRuntime.Containers, 1, "container should be created") + assert.Len(t, fakeImage.Images, 1, "image should be pulled") + for _, c := range fakeRuntime.Containers { + assert.Equal(t, runtimeapi.ContainerState_CONTAINER_RUNNING, c.State, "container should be running") + } + } + + if test.deviceAllocationShouldErr { + require.Empty(t, fakeRuntime.Sandboxes, "sandbox should not be created when device allocation fails") + require.Empty(t, fakeRuntime.Containers, "containers should not be created when device allocation fails") + } + }) + } +} + +// TestOnPodSandboxReadyTiming tests that OnPodSandboxReady is invoked after sandbox +// creation and network setup but before image pulling. +func TestOnPodSandboxReadyTiming(t *testing.T) { + tCtx := ktesting.Init(t) + fakeRuntime, fakeImage, m, err := createTestRuntimeManager(tCtx) + require.NoError(t, err) + + // track the state of pod when OnPodSandboxReady is invoked + var sandboxCount int + var containerCount int + var imageCount int + + testHelper := &testRuntimeHelper{ + FakeRuntimeHelper: &containertest.FakeRuntimeHelper{}, + captureStateFunc: func() { + sandboxCount = len(fakeRuntime.Sandboxes) + containerCount = len(fakeRuntime.Containers) + imageCount = len(fakeImage.Images) + }, + } + + m.runtimeHelper = testHelper + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: "timing-test-pod", + Name: "timing-test", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Image: "busybox", + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + }, + } + + backOff := flowcontrol.NewBackOff(time.Second, time.Minute) + result := m.SyncPod(tCtx, pod, &kubecontainer.PodStatus{}, []v1.Secret{}, backOff, false) + require.NoError(t, result.Error()) + + // verify the order that OnPodSandboxReady should be invoked after sandbox creation but before containers + assert.Equal(t, 1, sandboxCount, "sandbox should exist when OnPodSandboxReady is invoked") + assert.Equal(t, 0, containerCount, "containers should not exist yet when OnPodSandboxReady is invoked") + // Note that image may or may not be pulled at OnPodSandboxReady time depending on whether image exists + t.Logf("At OnPodSandboxReady time: sandboxes=%d, containers=%d, images=%d", sandboxCount, containerCount, imageCount) + + // verify the final state of pod + assert.Len(t, fakeRuntime.Sandboxes, 1, "final sandbox count") + assert.Len(t, fakeRuntime.Containers, 1, "final container count") +} From 098f2c6ec6067369205fcdfaf8c735f47140a334 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Tue, 20 Jan 2026 19:48:18 +0530 Subject: [PATCH 3/3] add e2e tests verifying PodReadyToStartContainers condition set using criProxy to inject delay time --- .../e2e_node/pod_conditions_criproxy_linux.go | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 test/e2e_node/pod_conditions_criproxy_linux.go diff --git a/test/e2e_node/pod_conditions_criproxy_linux.go b/test/e2e_node/pod_conditions_criproxy_linux.go new file mode 100644 index 00000000000..4ae93d57da9 --- /dev/null +++ b/test/e2e_node/pod_conditions_criproxy_linux.go @@ -0,0 +1,158 @@ +//go:build linux + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/features" + kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/test/e2e/feature" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/e2e_node/criproxy" + imageutils "k8s.io/kubernetes/test/utils/image" + admissionapi "k8s.io/pod-security-admission/api" +) + +// PodReadyToStartContainers condition timing tests with CRI Proxy delays (Linux only) +var _ = SIGDescribe("Pod conditions managed by Kubelet", func() { + f := framework.NewDefaultFramework("pod-conditions") + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + + f.Context("including PodReadyToStartContainers condition", f.WithSerial(), framework.WithFeatureGate(features.PodReadyToStartContainersCondition), func() { + tempSetCurrentKubeletConfig(f, func(ctx context.Context, initialConfig *kubeletconfig.KubeletConfiguration) { + if initialConfig.FeatureGates == nil { + initialConfig.FeatureGates = map[string]bool{} + } + }) + + f.Context("timing with CRI Proxy delays", feature.CriProxy, func() { + ginkgo.BeforeEach(func() { + if e2eCriProxy == nil { + ginkgo.Skip("Skip the test since the CRI Proxy is undefined. Please run with --cri-proxy-enabled=true") + } + if err := resetCRIProxyInjector(e2eCriProxy); err != nil { + ginkgo.Skip("Skip the test since the CRI Proxy is undefined.") + } + ginkgo.DeferCleanup(func() error { + return resetCRIProxyInjector(e2eCriProxy) + }) + }) + + ginkgo.It("a pod without init containers should report PodReadyToStartContainers condition set before image pull completes", runPodReadyToStartContainersTimingTest(f, false)) + ginkgo.It("a pod with init containers should report PodReadyToStartContainers condition set before image pull completes", runPodReadyToStartContainersTimingTest(f, true)) + }) + + addAfterEachForCleaningUpPods(f) + }) +}) + +// newPullImageAlwaysPodWithInitContainer creates a pod with init container and ImagePullPolicy: Always +func newPullImageAlwaysPodWithInitContainer() *v1.Pod { + podName := "cri-proxy-test-" + string(uuid.NewUUID()) + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Name: "init", + ImagePullPolicy: v1.PullAlways, + Command: []string{"sh", "-c", "sleep 2"}, + }, + }, + Containers: []v1.Container{ + { + Image: imageutils.GetPauseImageName(), + Name: "main", + ImagePullPolicy: v1.PullAlways, + }, + }, + }, + } + return pod +} + +func runPodReadyToStartContainersTimingTest(f *framework.Framework, hasInitContainers bool) func(ctx context.Context) { + return func(ctx context.Context) { + const delayTime = 15 * time.Second + + ginkgo.By("Injecting delay into PullImage calls") + err := addCRIProxyInjector(e2eCriProxy, func(apiName string) error { + if apiName == criproxy.PullImage { + ginkgo.By(fmt.Sprintf("Delaying PullImage by %v", delayTime)) + time.Sleep(delayTime) + } + return nil + }) + framework.ExpectNoError(err) + + ginkgo.By("Creating test pod with ImagePullPolicy: Always") + var testPod *v1.Pod + if hasInitContainers { + testPod = newPullImageAlwaysPodWithInitContainer() + } else { + testPod = newPullImageAlwaysPod() + } + testPod = e2epod.NewPodClient(f).Create(ctx, testPod) + + ginkgo.By("Waiting for PodReadyToStartContainers condition to be set") + gomega.Eventually(func() error { + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, testPod.Name, metav1.GetOptions{}) + if err != nil { + return err + } + _, err = getTransitionTimeForPodConditionWithStatus(pod, v1.PodReadyToStartContainers, true) + return err + }).WithPolling(500*time.Millisecond).WithTimeout(10*time.Second).Should(gomega.Succeed(), + "PodReadyToStartContainers condition should be set to True within %v", 10*time.Second) + + ginkgo.By("Verifying condition timing, it should be set quickly before image pull delay") + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, testPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + conditionTime, err := getTransitionTimeForPodConditionWithStatus(pod, v1.PodReadyToStartContainers, true) + framework.ExpectNoError(err) + + ginkgo.By("Waiting for pod to eventually become Running after image pull") + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, testPod)) + + ginkgo.By("Verifying condition was set before image pull completed") + pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, testPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + podReadyTime, err := getTransitionTimeForPodConditionWithStatus(pod, v1.PodReady, true) + framework.ExpectNoError(err) + + gomega.Expect(conditionTime.Before(podReadyTime)).To(gomega.BeTrueBecause( + "PodReadyToStartContainers was set at %v but PodReady was set at %v - condition should be set before image pull completes", + conditionTime, podReadyTime)) + } +}