Merge pull request #136477 from everettraven/feature/liveness-probe-fails-on-loopback-cert-expiry

Add loopback certificate expiration health check

pkg/controlplane/apiserver/config.go

@@ -131,7 +131,7 @@ func BuildGenericConfig(
 		return
 	}
 
-	if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil {
+	if lastErr = s.SecureServing.ApplyToConfig(genericConfig); lastErr != nil {
 		return
 	}
 

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -573,6 +573,18 @@ func (c *Config) AddHealthChecks(healthChecks ...healthz.HealthChecker) {
 	c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
 }
 
+// AddHealthzChecks adds the provided health checks to our config to be exposed by the
+// healthz endpoint of our configured apiserver.
+func (c *Config) AddHealthzChecks(healthChecks ...healthz.HealthChecker) {
+	c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
+}
+
+// AddLivezChecks adds the provided health checks to our config to be exposed by the
+// livez endpoint of our configured apiserver.
+func (c *Config) AddLivezChecks(healthChecks ...healthz.HealthChecker) {
+	c.LivezChecks = append(c.LivezChecks, healthChecks...)
+}
+
 // AddReadyzChecks adds a health check to our config to be exposed by the readyz endpoint
 // of our configured apiserver.
 func (c *Config) AddReadyzChecks(healthChecks ...healthz.HealthChecker) {

staging/src/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -105,7 +105,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
 		return err
 	}
-	if err := o.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
+	if err := o.SecureServing.ApplyToConfig(&config.Config); err != nil {
 		return err
 	}
 	if err := o.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {

staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go

@@ -18,26 +18,48 @@ package options
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/google/uuid"
 
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/rest"
 	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/utils/clock"
 )
 
 type SecureServingOptionsWithLoopback struct {
 	*SecureServingOptions
+	clock clock.PassiveClock
 }
 
 func (o *SecureServingOptions) WithLoopback() *SecureServingOptionsWithLoopback {
-	return &SecureServingOptionsWithLoopback{o}
+	return &SecureServingOptionsWithLoopback{
+		SecureServingOptions: o,
+		clock:                clock.RealClock{},
+	}
 }
 
+// Set a validity period of approximately 3 years for the loopback certificate
+// to avoid kube-apiserver disruptions due to certificate expiration.
+// When this certificate expires, restarting kube-apiserver will automatically
+// regenerate a new certificate with fresh validity dates.
+const maxAge = (3*365 + 1) * 24 * time.Hour
+
 // ApplyTo fills up serving information in the server configuration.
 func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.SecureServingInfo, loopbackClientConfig **rest.Config) error {
+	return s.applyTo(secureServingInfo, loopbackClientConfig, nil)
+}
+
+type HealthzLivezHealthChecksAdder interface {
+	AddHealthzChecks(checks ...healthz.HealthChecker)
+	AddLivezChecks(checks ...healthz.HealthChecker)
+}
+
+func (s *SecureServingOptionsWithLoopback) applyTo(secureServingInfo **server.SecureServingInfo, loopbackClientConfig **rest.Config, healthCheckAdder HealthzLivezHealthChecksAdder) error {
 	if s == nil || s.SecureServingOptions == nil || secureServingInfo == nil {
 		return nil
 	}
@@ -50,12 +72,6 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se
 		return nil
 	}
 
-	// Set a validity period of approximately 3 years for the loopback certificate
-	// to avoid kube-apiserver disruptions due to certificate expiration.
-	// When this certificate expires, restarting kube-apiserver will automatically
-	// regenerate a new certificate with fresh validity dates.
-	maxAge := (3*365 + 1) * 24 * time.Hour
-
 	// create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and
 	// let the server return it when the loopback client connects.
 	certPem, keyPem, err := certutil.GenerateSelfSignedCertKeyWithOptions(certutil.SelfSignedCertKeyOptions{
@@ -85,7 +101,37 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se
 
 	default:
 		*loopbackClientConfig = secureLoopbackClientConfig
+		if healthCheckAdder != nil {
+			s.addLoopbackServingCertificateHealthCheck(healthCheckAdder)
+		}
 	}
 
 	return nil
 }
+
+func (s *SecureServingOptionsWithLoopback) ApplyToConfig(cfg *server.Config) error {
+	return s.applyTo(&cfg.SecureServing, &cfg.LoopbackClientConfig, cfg)
+}
+
+// addLoopbackServingCertificateHealthCheck adds a health check called `loopback-certificate-expiry` to the
+// server that fails when the loopback client certificate has expired, enabling
+// liveness probes to be used to automatically restart the apiserver.
+func (s *SecureServingOptionsWithLoopback) addLoopbackServingCertificateHealthCheck(healthCheckAdder HealthzLivezHealthChecksAdder) {
+	expirationDate := s.clock.Now().Add(maxAge)
+	check := healthz.NamedCheck("loopback-serving-certificate", func(r *http.Request) error {
+		if s.clock.Now().After(expirationDate) {
+			return LoopbackCertificateExpiredError{}
+		}
+
+		return nil
+	})
+
+	healthCheckAdder.AddHealthzChecks(check)
+	healthCheckAdder.AddLivezChecks(check)
+}
+
+type LoopbackCertificateExpiredError struct{}
+
+func (lcee LoopbackCertificateExpiredError) Error() string {
+	return "loopback serving certificate is expired"
+}

staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback_test.go

@@ -17,11 +17,17 @@ limitations under the License.
 package options
 
 import (
+	"errors"
 	"net"
 	"testing"
+	"time"
 
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/rest"
+	clocktesting "k8s.io/utils/clock/testing"
 	netutils "k8s.io/utils/net"
 )
 
@@ -50,3 +56,81 @@ func TestEmptyMainCert(t *testing.T) {
 		t.Errorf("expected %d SNICert, got %d", e, a)
 	}
 }
+
+func TestApplyToConfig(t *testing.T) {
+	type testcase struct {
+		name    string
+		expired bool
+	}
+
+	testcases := []testcase{
+		{
+			name: "current time not after certificate expiry",
+		},
+		{
+			name:    "current time after certificate expiry",
+			expired: true,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := server.NewConfig(serializer.NewCodecFactory(runtime.NewScheme()))
+
+			ssowlb := (&SecureServingOptions{
+				BindAddress: netutils.ParseIPSloppy("127.0.0.1"),
+			}).WithLoopback()
+			now := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.Local)
+			fakeClock := clocktesting.NewFakeClock(now)
+			ssowlb.clock = fakeClock
+
+			ln, err := net.Listen("tcp", "127.0.0.1:0")
+			if err != nil {
+				t.Fatalf("failed to listen on 127.0.0.1:0")
+			}
+			defer ln.Close() //nolint:errcheck
+			ssowlb.Listener = ln
+			ssowlb.BindPort = ln.Addr().(*net.TCPAddr).Port
+
+			if err := ssowlb.ApplyToConfig(cfg); err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			if tc.expired {
+				fakeClock.Step(maxAge + 1)
+			}
+
+			checkSet := map[string][]healthz.HealthChecker{
+				"/livez":   cfg.LivezChecks,
+				"/healthz": cfg.HealthzChecks,
+			}
+
+			for endpoint, checks := range checkSet {
+				var foundChecker healthz.HealthChecker
+
+				for _, check := range checks {
+					if check.Name() == "loopback-serving-certificate" {
+						foundChecker = check
+						break
+					}
+				}
+
+				if foundChecker == nil {
+					t.Fatalf("loopback-serving-certificate health checker not found for %s endpoint", endpoint)
+				}
+
+				err = foundChecker.Check(nil)
+				if tc.expired {
+					if !errors.Is(err, LoopbackCertificateExpiredError{}) {
+						t.Errorf("%s endpoint check expected error and received error do not match. expected: %v , received: %v", endpoint, LoopbackCertificateExpiredError{}, err)
+					}
+					return
+				}
+
+				if err != nil {
+					t.Errorf("%s endpoint check received an unexpected error: %v", endpoint, err)
+				}
+			}
+		})
+	}
+}

test/integration/apiserver/health_handlers_test.go

@@ -59,6 +59,28 @@ func TestHealthHandler(t *testing.T) {
 	}
 }
 
+func TestHealthHandlerLoopbackServingCertificate(t *testing.T) {
+	_, c, _, teardownFn := setup(t)
+	defer teardownFn()
+
+	paths := []string{
+		"/healthz",
+		"/livez",
+	}
+
+	for _, path := range paths {
+		raw := readinessCheck(t, c, path, "")
+		if !strings.Contains(string(raw), "[+]loopback-serving-certificate ok") {
+			t.Errorf("%s result should contain loopback-serving-certificate ok. Raw: %v", path, string(raw))
+		}
+	}
+
+	raw := readinessCheck(t, c, "/readyz", "")
+	if strings.Contains(string(raw), "[+]loopback-serving-certificate ok") {
+		t.Errorf("/readyz result should not contain loopback-serving-certificate ok. Raw: %v", string(raw))
+	}
+}
+
 func readinessCheck(t *testing.T, c kubernetes.Interface, path string, exclude string) []byte {
 	var statusCode int
 	req := c.CoreV1().RESTClient().Get().AbsPath(path)