diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 2cde5fc0a9c..9821880adeb 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - itest: [certrotation, etcdrestore, localstorage, startup, custometcdargs, etcdsnapshot, kubeflags, longhorn, secretsencryption, flannelnone] + itest: [certrotation, cacertrotation, etcdrestore, localstorage, startup, custometcdargs, etcdsnapshot, kubeflags, longhorn, secretsencryption, flannelnone] max-parallel: 3 steps: - name: Checkout @@ -56,7 +56,7 @@ jobs: run: | chmod +x ./dist/artifacts/k3s mkdir -p $GOCOVERDIR - sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration + sudo -E env "PATH=$PATH" go test -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration -ginkgo.v -test.v - name: On Failure, Launch Debug Session uses: lhotari/action-upterm@v1 if: ${{ failure() }} @@ -71,4 +71,4 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./${{ matrix.itest }}.out flags: inttests # optional - verbose: true # optional (default = false) \ No newline at end of file + verbose: true # optional (default = false) diff --git a/scripts/build-tests-sonobuoy b/scripts/build-tests-sonobuoy index a016e6a4e4c..3cae271b4ba 100755 --- a/scripts/build-tests-sonobuoy +++ b/scripts/build-tests-sonobuoy @@ -14,7 +14,7 @@ PKG_TO_TEST=$(find ./pkg/ -type f -name "*_int_test.go" | sed -r 's|/[^/]+$||' | for i in $PKG_TO_TEST; do name=$(echo "${i##*/}") echo $name - go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration + go test -c -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration -ginkgo.v -test.v done # Integration tests under /tests @@ -22,7 +22,7 @@ PKG_TO_TEST=$(find ./tests/integration -type f -name "*_int_test.go" | sed -r 's for i in $PKG_TO_TEST; do name=$(echo "${i##*/}") echo $name - go test -c -v -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration + go test -c -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" -o dist/artifacts/k3s-integration-$name.test $i -run Integration -ginkgo.v -test.v done docker build -f ./tests/integration/Dockerfile.test -t $REPO . docker save $REPO -o ./dist/artifacts/$REPO.tar diff --git a/tests/integration/README.md b/tests/integration/README.md index d5c80516631..04e2313c3ac 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -22,7 +22,7 @@ See the [local storage test](../tests/integration/localstorage/localstorage_int_ Integration tests can be run with no k3s cluster present, each test will spin up and kill the appropriate k3s server it needs. Note: Integration tests must be run as root, prefix the commands below with `sudo -E env "PATH=$PATH"` if a sudo user. ```bash -go test ./tests/integration/... -run Integration +go test ./tests/integration/... -run Integration -ginkgo.v -test.v ``` Additionally, to generate JUnit reporting for the tests, the Ginkgo CLI is used @@ -32,7 +32,7 @@ ginkgo --junit-report=result.xml ./tests/integration/... Integration tests can be run on an existing single-node cluster via compile time flag, tests will skip if the server is not configured correctly. ```bash -go test -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" ./tests/integration/... -run Integration +go test -ldflags "-X 'github.com/k3s-io/k3s/tests/integration.existingServer=True'" ./tests/integration/... -run Integration -ginkgo.v -test.v ``` Integration tests can also be run via a [Sonobuoy](https://sonobuoy.io/docs/v0.53.2/) plugin on an existing single-node cluster. diff --git a/tests/integration/cacertrotation/cacertrotation_int_test.go b/tests/integration/cacertrotation/cacertrotation_int_test.go new file mode 100644 index 00000000000..31df7369a77 --- /dev/null +++ b/tests/integration/cacertrotation/cacertrotation_int_test.go @@ -0,0 +1,108 @@ +package ca_cert_rotation_test + +import ( + "fmt" + "strings" + "testing" + + testutil "github.com/k3s-io/k3s/tests/integration" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const tmpdDataDir = "/tmp/cacertrotationtest" + +var server, server2 *testutil.K3sServer +var serverArgs = []string{"--cluster-init", "-t", "test", "-d", tmpdDataDir} +var certHash, caCertHash string +var testLock int + +var _ = BeforeSuite(func() { + if !testutil.IsExistingServer() { + var err error + testLock, err = testutil.K3sTestLock() + Expect(err).ToNot(HaveOccurred()) + server, err = testutil.K3sStartServer(serverArgs...) + Expect(err).ToNot(HaveOccurred()) + } +}) + +var _ = Describe("ca certificate rotation", Ordered, func() { + BeforeEach(func() { + if testutil.IsExistingServer() && !testutil.ServerArgsPresent(serverArgs) { + Skip("Test needs k3s server with: " + strings.Join(serverArgs, " ")) + } + }) + When("a new server is created", func() { + It("starts up with no problems", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "180s", "5s").Should(Succeed()) + }) + It("get certificate hash", func() { + // get md5sum of the CA certs + var err error + caCertHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '") + Expect(err).ToNot(HaveOccurred()) + certHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/serving-kube-apiserver.crt | cut -f 1 -d' '") + Expect(err).ToNot(HaveOccurred()) + }) + It("generates updated ca-certificates", func() { + cmd := fmt.Sprintf("DATA_DIR=%s ../../../contrib/util/rotate-default-ca-certs.sh", tmpdDataDir) + By("running command: " + cmd) + res, err := testutil.RunCommand(cmd) + By("checking command results: " + res) + Expect(err).ToNot(HaveOccurred()) + }) + It("certificate rotate-ca", func() { + res, err := testutil.K3sCmd("certificate", "rotate-ca", "-d", tmpdDataDir, "--path", tmpdDataDir+"/server/rotate-ca") + By("checking command results: " + res) + Expect(err).ToNot(HaveOccurred()) + }) + It("stop k3s", func() { + Expect(testutil.K3sKillServer(server)).To(Succeed()) + }) + It("start k3s server", func() { + var err error + server2, err = testutil.K3sStartServer(serverArgs...) + Expect(err).ToNot(HaveOccurred()) + }) + It("starts up with no problems", func() { + Eventually(func() error { + return testutil.K3sDefaultDeployments() + }, "360s", "5s").Should(Succeed()) + }) + It("get certificate hash", func() { + // get md5sum of the CA certs + var err error + caCertHashAfter, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '") + Expect(err).ToNot(HaveOccurred()) + certHashAfter, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/serving-kube-apiserver.crt | cut -f 1 -d' '") + Expect(err).ToNot(HaveOccurred()) + Expect(certHash).To(Not(Equal(certHashAfter))) + Expect(caCertHash).To(Not(Equal(caCertHashAfter))) + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if !testutil.IsExistingServer() { + if failed { + testutil.K3sSaveLog(server, false) + } + Expect(testutil.K3sKillServer(server)).To(Succeed()) + Expect(testutil.K3sCleanup(-1, "")).To(Succeed()) + Expect(testutil.K3sKillServer(server2)).To(Succeed()) + Expect(testutil.K3sCleanup(testLock, tmpdDataDir)).To(Succeed()) + } +}) + +func Test_IntegrationCertRotation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CA Cert rotation Suite") +} diff --git a/tests/integration/integration.go b/tests/integration/integration.go index 6f6745dbcb6..42b775a4921 100644 --- a/tests/integration/integration.go +++ b/tests/integration/integration.go @@ -389,9 +389,10 @@ func RunCommand(cmd string) (string, error) { c := exec.Command("bash", "-c", cmd) var out bytes.Buffer c.Stdout = &out + c.Stderr = &out err := c.Run() if err != nil { - return "", fmt.Errorf("%s", err) + return out.String(), fmt.Errorf("%s", err) } return out.String(), nil }