diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 77e723bdbf4..dec22de82fe 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -131,7 +131,7 @@ func BuildGenericConfig( return } - if lastErr = s.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil { + if lastErr = s.SecureServing.ApplyToConfig(genericConfig); lastErr != nil { return } diff --git a/staging/src/k8s.io/apiserver/pkg/server/config.go b/staging/src/k8s.io/apiserver/pkg/server/config.go index d10415b945f..a4b131d60a7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/config.go @@ -573,6 +573,18 @@ func (c *Config) AddHealthChecks(healthChecks ...healthz.HealthChecker) { c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...) } +// AddHealthzChecks adds the provided health checks to our config to be exposed by the +// healthz endpoint of our configured apiserver. +func (c *Config) AddHealthzChecks(healthChecks ...healthz.HealthChecker) { + c.HealthzChecks = append(c.HealthzChecks, healthChecks...) +} + +// AddLivezChecks adds the provided health checks to our config to be exposed by the +// livez endpoint of our configured apiserver. +func (c *Config) AddLivezChecks(healthChecks ...healthz.HealthChecker) { + c.LivezChecks = append(c.LivezChecks, healthChecks...) +} + // AddReadyzChecks adds a health check to our config to be exposed by the readyz endpoint // of our configured apiserver. func (c *Config) AddReadyzChecks(healthChecks ...healthz.HealthChecker) { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go index a80c3f9ed42..9721797787a 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/recommended.go @@ -105,7 +105,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil { return err } - if err := o.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil { + if err := o.SecureServing.ApplyToConfig(&config.Config); err != nil { return err } if err := o.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go index 980ddc61a48..2dea7d2db19 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go @@ -18,26 +18,48 @@ package options import ( "fmt" + "net/http" "time" "github.com/google/uuid" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/server/healthz" "k8s.io/client-go/rest" certutil "k8s.io/client-go/util/cert" + "k8s.io/utils/clock" ) type SecureServingOptionsWithLoopback struct { *SecureServingOptions + clock clock.PassiveClock } func (o *SecureServingOptions) WithLoopback() *SecureServingOptionsWithLoopback { - return &SecureServingOptionsWithLoopback{o} + return &SecureServingOptionsWithLoopback{ + SecureServingOptions: o, + clock: clock.RealClock{}, + } } +// Set a validity period of approximately 3 years for the loopback certificate +// to avoid kube-apiserver disruptions due to certificate expiration. +// When this certificate expires, restarting kube-apiserver will automatically +// regenerate a new certificate with fresh validity dates. +const maxAge = (3*365 + 1) * 24 * time.Hour + // ApplyTo fills up serving information in the server configuration. func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.SecureServingInfo, loopbackClientConfig **rest.Config) error { + return s.applyTo(secureServingInfo, loopbackClientConfig, nil) +} + +type HealthzLivezHealthChecksAdder interface { + AddHealthzChecks(checks ...healthz.HealthChecker) + AddLivezChecks(checks ...healthz.HealthChecker) +} + +func (s *SecureServingOptionsWithLoopback) applyTo(secureServingInfo **server.SecureServingInfo, loopbackClientConfig **rest.Config, healthCheckAdder HealthzLivezHealthChecksAdder) error { if s == nil || s.SecureServingOptions == nil || secureServingInfo == nil { return nil } @@ -50,12 +72,6 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se return nil } - // Set a validity period of approximately 3 years for the loopback certificate - // to avoid kube-apiserver disruptions due to certificate expiration. - // When this certificate expires, restarting kube-apiserver will automatically - // regenerate a new certificate with fresh validity dates. - maxAge := (3*365 + 1) * 24 * time.Hour - // create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and // let the server return it when the loopback client connects. certPem, keyPem, err := certutil.GenerateSelfSignedCertKeyWithOptions(certutil.SelfSignedCertKeyOptions{ @@ -85,7 +101,37 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se default: *loopbackClientConfig = secureLoopbackClientConfig + if healthCheckAdder != nil { + s.addLoopbackServingCertificateHealthCheck(healthCheckAdder) + } } return nil } + +func (s *SecureServingOptionsWithLoopback) ApplyToConfig(cfg *server.Config) error { + return s.applyTo(&cfg.SecureServing, &cfg.LoopbackClientConfig, cfg) +} + +// addLoopbackServingCertificateHealthCheck adds a health check called `loopback-certificate-expiry` to the +// server that fails when the loopback client certificate has expired, enabling +// liveness probes to be used to automatically restart the apiserver. +func (s *SecureServingOptionsWithLoopback) addLoopbackServingCertificateHealthCheck(healthCheckAdder HealthzLivezHealthChecksAdder) { + expirationDate := s.clock.Now().Add(maxAge) + check := healthz.NamedCheck("loopback-serving-certificate", func(r *http.Request) error { + if s.clock.Now().After(expirationDate) { + return LoopbackCertificateExpiredError{} + } + + return nil + }) + + healthCheckAdder.AddHealthzChecks(check) + healthCheckAdder.AddLivezChecks(check) +} + +type LoopbackCertificateExpiredError struct{} + +func (lcee LoopbackCertificateExpiredError) Error() string { + return "loopback serving certificate is expired" +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback_test.go index c4b0c57b568..95bb1122974 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/serving_with_loopback_test.go @@ -17,11 +17,17 @@ limitations under the License. package options import ( + "errors" "net" "testing" + "time" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" "k8s.io/client-go/rest" + clocktesting "k8s.io/utils/clock/testing" netutils "k8s.io/utils/net" ) @@ -50,3 +56,81 @@ func TestEmptyMainCert(t *testing.T) { t.Errorf("expected %d SNICert, got %d", e, a) } } + +func TestApplyToConfig(t *testing.T) { + type testcase struct { + name string + expired bool + } + + testcases := []testcase{ + { + name: "current time not after certificate expiry", + }, + { + name: "current time after certificate expiry", + expired: true, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + cfg := server.NewConfig(serializer.NewCodecFactory(runtime.NewScheme())) + + ssowlb := (&SecureServingOptions{ + BindAddress: netutils.ParseIPSloppy("127.0.0.1"), + }).WithLoopback() + now := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.Local) + fakeClock := clocktesting.NewFakeClock(now) + ssowlb.clock = fakeClock + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen on 127.0.0.1:0") + } + defer ln.Close() //nolint:errcheck + ssowlb.Listener = ln + ssowlb.BindPort = ln.Addr().(*net.TCPAddr).Port + + if err := ssowlb.ApplyToConfig(cfg); err != nil { + t.Errorf("unexpected error: %v", err) + } + + if tc.expired { + fakeClock.Step(maxAge + 1) + } + + checkSet := map[string][]healthz.HealthChecker{ + "/livez": cfg.LivezChecks, + "/healthz": cfg.HealthzChecks, + } + + for endpoint, checks := range checkSet { + var foundChecker healthz.HealthChecker + + for _, check := range checks { + if check.Name() == "loopback-serving-certificate" { + foundChecker = check + break + } + } + + if foundChecker == nil { + t.Fatalf("loopback-serving-certificate health checker not found for %s endpoint", endpoint) + } + + err = foundChecker.Check(nil) + if tc.expired { + if !errors.Is(err, LoopbackCertificateExpiredError{}) { + t.Errorf("%s endpoint check expected error and received error do not match. expected: %v , received: %v", endpoint, LoopbackCertificateExpiredError{}, err) + } + return + } + + if err != nil { + t.Errorf("%s endpoint check received an unexpected error: %v", endpoint, err) + } + } + }) + } +} diff --git a/test/integration/apiserver/health_handlers_test.go b/test/integration/apiserver/health_handlers_test.go index 7199d5a1ea2..1225d682d46 100644 --- a/test/integration/apiserver/health_handlers_test.go +++ b/test/integration/apiserver/health_handlers_test.go @@ -59,6 +59,28 @@ func TestHealthHandler(t *testing.T) { } } +func TestHealthHandlerLoopbackServingCertificate(t *testing.T) { + _, c, _, teardownFn := setup(t) + defer teardownFn() + + paths := []string{ + "/healthz", + "/livez", + } + + for _, path := range paths { + raw := readinessCheck(t, c, path, "") + if !strings.Contains(string(raw), "[+]loopback-serving-certificate ok") { + t.Errorf("%s result should contain loopback-serving-certificate ok. Raw: %v", path, string(raw)) + } + } + + raw := readinessCheck(t, c, "/readyz", "") + if strings.Contains(string(raw), "[+]loopback-serving-certificate ok") { + t.Errorf("/readyz result should not contain loopback-serving-certificate ok. Raw: %v", string(raw)) + } +} + func readinessCheck(t *testing.T, c kubernetes.Interface, path string, exclude string) []byte { var statusCode int req := c.CoreV1().RESTClient().Get().AbsPath(path)