Add ca-cert rotation integration test

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

.github/workflows/integration.yaml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        itest: [certrotation, etcdrestore, localstorage, startup, custometcdargs, etcdsnapshot, kubeflags, longhorn, secretsencryption, flannelnone]
+        itest: [certrotation, cacertrotation, etcdrestore, localstorage, startup, custometcdargs, etcdsnapshot, kubeflags, longhorn, secretsencryption, flannelnone]
       max-parallel: 3
     steps:
     - name: Checkout
@@ -56,7 +56,7 @@ jobs:
       run: | 
         chmod +x ./dist/artifacts/k3s
         mkdir -p $GOCOVERDIR
-        sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration
+        sudo -E env "PATH=$PATH" go test -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration -ginkgo.v -test.v
     - name: On Failure, Launch Debug Session
       uses: lhotari/action-upterm@v1
       if: ${{ failure() }}
@@ -71,4 +71,4 @@ jobs:
         token: ${{ secrets.CODECOV_TOKEN }}
         files: ./${{ matrix.itest }}.out
         flags: inttests # optional
-        verbose: true # optional (default = false)
+        verbose: true # optional (default = false)

scripts/build-tests-sonobuoy

@@ -14,7 +14,7 @@ PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' |
 for i in $PKG_TO_TEST; do
     name=$(echo "${i##*/}")
     echo $name
-    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+    go test -c -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration -ginkgo.v -test.v
 done
 
 # Integration tests under /tests
@@ -22,7 +22,7 @@ PKG_TO_TEST=$(find ./tests/integration -type f -name "*_int_test.go" | sed -r 's
 for i in $PKG_TO_TEST; do
     name=$(echo "${i##*/}")
     echo $name
-    go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration
+    go test -c -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration -ginkgo.v -test.v
 done
 docker build -f ./tests/integration/Dockerfile.test -t $REPO .
 docker save $REPO -o ./dist/artifacts/$REPO.tar

tests/integration/README.md

@@ -22,7 +22,7 @@ See the [local storage test](../tests/integration/localstorage/localstorage_int_
 Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs.  
 Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user.
 ```bash
-go test ./tests/integration/... -run Integration
+go test ./tests/integration/... -run Integration -ginkgo.v -test.v
 ```
 
 Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used
@@ -32,7 +32,7 @@ ginkgo --junit-report=result.xml ./tests/integration/...
 
 Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly.
 ```bash
-go test -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" ./tests/integration/... -run Integration
+go test -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" ./tests/integration/... -run Integration -ginkgo.v -test.v
 ```
 
 Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster.

tests/integration/cacertrotation/cacertrotation_int_test.go

@@ -0,0 +1,108 @@
+package ca_cert_rotation_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	testutil "github.com/k3s-io/k3s/tests/integration"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const tmpdDataDir = "/tmp/cacertrotationtest"
+
+var server, server2 *testutil.K3sServer
+var serverArgs = []string{"--cluster-init", "-t", "test", "-d", tmpdDataDir}
+var certHash, caCertHash string
+var testLock int
+
+var _ = BeforeSuite(func() {
+	if !testutil.IsExistingServer() {
+		var err error
+		testLock, err = testutil.K3sTestLock()
+		Expect(err).ToNot(HaveOccurred())
+		server, err = testutil.K3sStartServer(serverArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
+})
+
+var _ = Describe("ca certificate rotation", Ordered, func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) {
+			Skip("Test needs k3s server with: " + strings.Join(serverArgs, " "))
+		}
+	})
+	When("a new server is created", func() {
+		It("starts up with no problems", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "180s", "5s").Should(Succeed())
+		})
+		It("get certificate hash", func() {
+			// get md5sum of the CA certs
+			var err error
+			caCertHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '")
+			Expect(err).ToNot(HaveOccurred())
+			certHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/serving-kube-apiserver.crt | cut -f 1 -d' '")
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("generates updated ca-certificates", func() {
+			cmd := fmt.Sprintf("DATA_DIR=%s ../../../contrib/util/rotate-default-ca-certs.sh", tmpdDataDir)
+			By("running command: " + cmd)
+			res, err := testutil.RunCommand(cmd)
+			By("checking command results: " + res)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("certificate rotate-ca", func() {
+			res, err := testutil.K3sCmd("certificate", "rotate-ca", "-d", tmpdDataDir, "--path", tmpdDataDir+"/server/rotate-ca")
+			By("checking command results: " + res)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("stop k3s", func() {
+			Expect(testutil.K3sKillServer(server)).To(Succeed())
+		})
+		It("start k3s server", func() {
+			var err error
+			server2, err = testutil.K3sStartServer(serverArgs...)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("starts up with no problems", func() {
+			Eventually(func() error {
+				return testutil.K3sDefaultDeployments()
+			}, "360s", "5s").Should(Succeed())
+		})
+		It("get certificate hash", func() {
+			// get md5sum of the CA certs
+			var err error
+			caCertHashAfter, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '")
+			Expect(err).ToNot(HaveOccurred())
+			certHashAfter, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/serving-kube-apiserver.crt | cut -f 1 -d' '")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(certHash).To(Not(Equal(certHashAfter)))
+			Expect(caCertHash).To(Not(Equal(caCertHashAfter)))
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		if failed {
+			testutil.K3sSaveLog(server, false)
+		}
+		Expect(testutil.K3sKillServer(server)).To(Succeed())
+		Expect(testutil.K3sCleanup(-1, "")).To(Succeed())
+		Expect(testutil.K3sKillServer(server2)).To(Succeed())
+		Expect(testutil.K3sCleanup(testLock, tmpdDataDir)).To(Succeed())
+	}
+})
+
+func Test_IntegrationCertRotation(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "CA Cert rotation Suite")
+}

tests/integration/integration.go

@@ -389,9 +389,10 @@ func RunCommand(cmd string) (string, error) {
 	c := exec.Command("bash", "-c", cmd)
 	var out bytes.Buffer
 	c.Stdout = &out
+	c.Stderr = &out
 	err := c.Run()
 	if err != nil {
-		return "", fmt.Errorf("%s", err)
+		return out.String(), fmt.Errorf("%s", err)
 	}
 	return out.String(), nil
 }