From fd48cd623340a4a6e3b2717dede368283cedec1a Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 18 Dec 2025 15:54:49 -0800 Subject: [PATCH] Allow k3s secrets-encrypt enable on existing clusters - Places an identity provider as a setup to enable later encryption - Update secrets-encryption test Signed-off-by: Derek Nola --- pkg/cli/secretsencrypt/secrets_encrypt.go | 27 ++++---- pkg/daemons/control/deps/deps.go | 7 ++- pkg/secretsencrypt/config.go | 34 ++++++++++ pkg/server/handlers/secrets-encrypt.go | 15 ++++- .../secretsencryption_test.go | 63 +++++++++++++------ 5 files changed, 111 insertions(+), 35 deletions(-) diff --git a/pkg/cli/secretsencrypt/secrets_encrypt.go b/pkg/cli/secretsencrypt/secrets_encrypt.go index bdc2a96ec66..5383364542d 100644 --- a/pkg/cli/secretsencrypt/secrets_encrypt.go +++ b/pkg/cli/secretsencrypt/secrets_encrypt.go @@ -129,21 +129,22 @@ func Status(app *cli.Context) error { } else { statusOutput += fmt.Sprintf("Server Encryption Hashes: %s\n", status.HashError) } - var tabBuffer bytes.Buffer - w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0) - fmt.Fprint(w, "\n") - fmt.Fprint(w, "Active\tKey Type\tName\n") - fmt.Fprint(w, "------\t--------\t----\n") - if status.ActiveKey != "" { - ak := strings.Split(status.ActiveKey, " ") - fmt.Fprintf(w, " *\t%s\t%s\n", ak[0], ak[1]) + if status.ActiveKey != "" || len(status.InactiveKeys) > 0 { + w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0) + fmt.Fprint(w, "\n") + fmt.Fprint(w, "Active\tKey Type\tName\n") + fmt.Fprint(w, "------\t--------\t----\n") + if status.ActiveKey != "" { + ak := strings.Split(status.ActiveKey, " ") + fmt.Fprintf(w, " *\t%s\t%s\n", ak[0], ak[1]) + } + for _, k := range status.InactiveKeys { + ik := strings.Split(k, " ") + fmt.Fprintf(w, "\t%s\t%s\n", ik[0], ik[1]) + } + w.Flush() } - for _, k := range status.InactiveKeys { - ik := strings.Split(k, " ") - fmt.Fprintf(w, "\t%s\t%s\n", ik[0], ik[1]) - } - w.Flush() fmt.Println(statusOutput + tabBuffer.String()) return nil } diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index c96dfa0b9b6..d076e99eb58 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -774,7 +774,7 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error { case secretsencrypt.SecretBoxProvider: keyName = "secretboxkey" default: - return fmt.Errorf("unsupported secrets-encryption-key-type %s", controlConfig.EncryptProvider) + return fmt.Errorf("unsupported secrets-encryption-provider %s", controlConfig.EncryptProvider) } if s, err := os.Stat(runtime.EncryptionConfig); err == nil && s.Size() > 0 { // On upgrade from older versions, the encryption hash may not exist, create it @@ -801,7 +801,8 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error { }, } var provider []apiserverconfigv1.ProviderConfiguration - if controlConfig.EncryptProvider == secretsencrypt.AESCBCProvider { + switch controlConfig.EncryptProvider { + case secretsencrypt.AESCBCProvider: provider = []apiserverconfigv1.ProviderConfiguration{ { AESCBC: &apiserverconfigv1.AESConfiguration{ @@ -812,7 +813,7 @@ func genEncryptionConfigAndState(controlConfig *config.Control) error { Identity: &apiserverconfigv1.IdentityConfiguration{}, }, } - } else if controlConfig.EncryptProvider == secretsencrypt.SecretBoxProvider { + case secretsencrypt.SecretBoxProvider: provider = []apiserverconfigv1.ProviderConfiguration{ { Secretbox: &apiserverconfigv1.SecretboxConfiguration{ diff --git a/pkg/secretsencrypt/config.go b/pkg/secretsencrypt/config.go index c129fb75432..a0c8ccc8beb 100644 --- a/pkg/secretsencrypt/config.go +++ b/pkg/secretsencrypt/config.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "time" "github.com/k3s-io/k3s/pkg/daemons/config" @@ -174,6 +175,39 @@ func WriteEncryptionConfig(runtime *config.ControlRuntime, keys *EncryptionKeys, return util.AtomicWrite(runtime.EncryptionConfig, jsonfile, 0600) } +// WriteIdentityConfig creates an identity-only configuration for clusters that +// previously had no encryption config, effectively disabling encryption, but +// preparing a node for future reencryption. +func WriteIdentityConfig(control *config.Control) error { + providers := []apiserverconfigv1.ProviderConfiguration{ + { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }, + } + + encConfig := apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: providers, + }, + }, + } + jsonfile, err := json.Marshal(encConfig) + if err != nil { + return err + } + if control.Runtime.EncryptionConfig == "" { + control.Runtime.EncryptionConfig = filepath.Join(control.DataDir, "cred", "encryption-config.json") + } + logrus.Info("Enabling secrets encryption with identity provider, restart with secrets-encryption") + return util.AtomicWrite(control.Runtime.EncryptionConfig, jsonfile, 0600) +} + func GenEncryptionConfigHash(runtime *config.ControlRuntime) (string, error) { curEncryptionByte, err := os.ReadFile(runtime.EncryptionConfig) if err != nil { diff --git a/pkg/server/handlers/secrets-encrypt.go b/pkg/server/handlers/secrets-encrypt.go index 2cca9819c22..59cc55a185f 100644 --- a/pkg/server/handlers/secrets-encrypt.go +++ b/pkg/server/handlers/secrets-encrypt.go @@ -88,6 +88,8 @@ func encryptionStatus(control *config.Control) (EncryptionState, error) { } if providers[len(providers)-1].Identity != nil && (providers[0].AESCBC != nil || providers[0].Secretbox != nil) { state.Enable = ptr.To(true) + } else if control.EncryptSecrets && providers[0].Identity != nil && len(providers) == 1 { + state.Enable = ptr.To(false) } else if !control.EncryptSecrets || providers[0].Identity != nil && (providers[1].AESCBC != nil || providers[1].Secretbox != nil) { state.Enable = ptr.To(false) } @@ -137,7 +139,13 @@ func encryptionStatus(control *config.Control) (EncryptionState, error) { func encryptionEnable(ctx context.Context, control *config.Control, enable bool) error { providers, err := secretsencrypt.GetEncryptionProviders(control.Runtime) - if err != nil { + // Enable secrets encryption with an identity provider on a cluster that does not have any encryption config + if err != nil && os.IsNotExist(err) && enable { + if err := secretsencrypt.WriteIdentityConfig(control); err != nil { + return err + } + return cluster.Save(ctx, control, true) + } else if err != nil { return err } if len(providers) > 3 { @@ -390,6 +398,7 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo // Remove old key. If there is only one of that key type, the cluster just // migrated between key types. Check for the other key type and remove that. + // If that key type type doesn't exist, we are switching from the identity provider, so no key is removed. curKeys, err := secretsencrypt.GetEncryptionKeys(control.Runtime) if err != nil { return err @@ -400,6 +409,8 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo if len(curKeys.AESCBCKeys) == 1 && len(curKeys.SBKeys) > 0 { logrus.Infoln("Removing secretbox key: ", curKeys.SBKeys[len(curKeys.SBKeys)-1]) curKeys.SBKeys = curKeys.SBKeys[:len(curKeys.SBKeys)-1] + } else if len(curKeys.AESCBCKeys) == 1 && curKeys.Identity { + logrus.Infoln("No keys to remove, switched from identity provider") } else { logrus.Infoln("Removing aescbc key: ", curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1]) curKeys.AESCBCKeys = curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1] @@ -408,6 +419,8 @@ func reencryptAndRemoveKey(ctx context.Context, control *config.Control, skip bo if len(curKeys.SBKeys) == 1 && len(curKeys.AESCBCKeys) > 0 { logrus.Infoln("Removing aescbc key: ", curKeys.AESCBCKeys[len(curKeys.AESCBCKeys)-1]) curKeys.AESCBCKeys = curKeys.AESCBCKeys[:len(curKeys.AESCBCKeys)-1] + } else if len(curKeys.SBKeys) == 1 && curKeys.Identity { + logrus.Infoln("No keys to remove, switched from identity provider") } else { logrus.Infoln("Removing secretbox key: ", curKeys.SBKeys[len(curKeys.SBKeys)-1]) curKeys.SBKeys = curKeys.SBKeys[:len(curKeys.SBKeys)-1] diff --git a/tests/docker/secretsencryption/secretsencryption_test.go b/tests/docker/secretsencryption/secretsencryption_test.go index 1254ad69811..0162641908b 100644 --- a/tests/docker/secretsencryption/secretsencryption_test.go +++ b/tests/docker/secretsencryption/secretsencryption_test.go @@ -28,7 +28,6 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { var err error tc, err = docker.NewTestConfig("rancher/systemd-node") Expect(err).NotTo(HaveOccurred()) - tc.ServerYaml = `secrets-encryption: true` Expect(tc.ProvisionServers(*serverCount)).To(Succeed()) Eventually(func() error { return tests.CheckDefaultDeployments(tc.KubeconfigFile) @@ -38,23 +37,48 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { }, "40s", "5s").Should(Succeed()) }) }) - Context("Secrets Keys are rotated:", func() { + Context("Secrets are added without encryption:", func() { It("Deploys several secrets", func() { _, err := tc.DeployWorkload("secrets.yaml") Expect(err).NotTo(HaveOccurred(), "Secrets not deployed") }) + It("Verifies encryption disabled", func() { + cmd := "k3s secrets-encrypt status" + for _, node := range tc.Servers { + res, err := node.RunCmdOnNode(cmd) + Expect(err).NotTo(HaveOccurred()) + Expect(res).Should(ContainSubstring("Encryption Status: Disabled, no configuration file found")) + } + }) + }) + Context("Secrets encryption is enabled on the cluster:", func() { + It("Enable secrets-encryption", func() { + cmd := "k3s secrets-encrypt enable" + Expect(tc.Servers[0].RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred()) + cmd = "echo 'secrets-encryption: true\n' >> /etc/rancher/k3s/config.yaml" + for _, node := range tc.Servers { + Expect(node.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred()) + } + }) + + It("Restarts K3s servers", func() { + Expect(docker.RestartCluster(tc.Servers)).To(Succeed()) + }) It("Verifies encryption start stage", func() { cmd := "k3s secrets-encrypt status" for _, node := range tc.Servers { - res, err := node.RunCmdOnNode(cmd) - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) - Expect(res).Should(ContainSubstring("Current Rotation Stage: start")) - Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + Eventually(func(g Gomega) { + res, err := node.RunCmdOnNode(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) + g.Expect(res).Should(ContainSubstring("Current Rotation Stage: start")) + g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) + }, "120s", "5s").Should(Succeed()) } }) - + }) + Context("Secrets Keys are rotated:", func() { It("Rotates the Secrets-Encryption Keys", func() { cmd := "k3s secrets-encrypt rotate-keys" res, err := tc.Servers[0].RunCmdOnNode(cmd) @@ -70,7 +94,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { } else { g.Expect(res).Should(ContainSubstring("Current Rotation Stage: start")) } - }, "420s", "10s").Should(Succeed()) + }, "240s", "10s").Should(Succeed()) } }) @@ -87,10 +111,9 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) - }, "420s", "2s").Should(Succeed()) + }, "240s", "2s").Should(Succeed()) } }) - }) Context("Disabling Secrets-Encryption", func() { @@ -113,7 +136,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { } else { g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) } - }, "420s", "2s").Should(Succeed()) + }, "240s", "2s").Should(Succeed()) } }) @@ -126,7 +149,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { for _, node := range tc.Servers { Eventually(func(g Gomega) { g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Disabled")) - }, "420s", "2s").Should(Succeed()) + }, "240s", "2s").Should(Succeed()) } }) @@ -152,7 +175,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { } else { g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) } - }, "420s", "2s").Should(Succeed()) + }, "240s", "2s").Should(Succeed()) } }) @@ -165,7 +188,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { for _, node := range tc.Servers { Eventually(func(g Gomega) { g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Enabled")) - }, "420s", "2s").Should(Succeed()) + }, "240s", "2s").Should(Succeed()) } }) }) @@ -195,7 +218,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { } else { g.Expect(res).Should(ContainSubstring("AES-CBC")) } - }, "420s", "10s").Should(Succeed()) + }, "240s", "10s").Should(Succeed()) } }) @@ -211,7 +234,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { g.Expect(err).NotTo(HaveOccurred()) g.Expect(res).Should(ContainSubstring("XSalsa20-POLY1305")) g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match")) - }, "420s", "2s").Should(Succeed()) + }, "240s", "2s").Should(Succeed()) } }) }) @@ -225,7 +248,11 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", docker.TailJournalLogs(1000, append(tc.Servers, tc.Agents...))) + log_length := 10 + if *ci { + log_length = 1000 + } + AddReportEntry("journald-logs", docker.TailJournalLogs(log_length, append(tc.Servers, tc.Agents...))) } if *ci || (tc != nil && !failed) { Expect(tc.Cleanup()).To(Succeed())