Build k3s overhaul (#12200)

* Add full ci support without Dapper
* Seperate git and other version tags, improves caching on binary builds
* Use new local targets for build-k3s.yaml workflow
* Allow optional ghcr build caching
* Build binary using GHA native commands
* Use internal setup-go action for e2e.yaml
* Add emulation builds to k3s-build.yaml (for arm32 and future riscv64)
* Be consistent in k3s artifact names
* Fix package/dockerfile warnings
* Fix install script for PR installs

Signed-off-by: Derek Nola <derek.nola@suse.com>

.drone.yml

@@ -576,7 +576,7 @@ steps:
   commands:
     - DOCKER_BUILDKIT=1 docker build --target test-e2e -t test-e2e -f Dockerfile.test .
     - apk add make git bash
-    - GOCOVER=1 make local
+    - GOCOVER=1 make local-binary
     - cp dist/artifacts/* /tmp/artifacts/
   volumes:
   - name: cache

.github/workflows/build-k3s.yaml

@@ -1,12 +1,10 @@
-name: Build K3s
-
 on: 
   workflow_call:
     inputs:
       arch:
         type: string
-        description: 'Architecture to build (ubuntu-latest or ubuntu-24.04-arm)'
-        default: 'ubuntu-latest'
+        description: 'Architecture to build (amd64, arm64, or arm)'
+        default: 'amd64'
       os:
         type: string
         description: 'Target OS (linux or windows)'
@@ -15,31 +13,117 @@ on:
         type: boolean
         required: false
         default: false
+      cache:
+        type: string
+        description: 'Cache mode: "read", "write", or empty for no cache'
+        required: false
+        default: ''
+
+# Note that is workflow requires the following permissions:
+#   contents: read
+# If using the cache: write option, you will need:
+#   packages: write
+# If using the cache: read option, you will need:
+#   packages: read
+
 
-permissions:
-  contents: read
 
 jobs:
   build:
-    name: Build K3s (${{ inputs.os }} on ${{ inputs.arch }})
-    runs-on: ${{ inputs.arch }}
+    name: Build # DO NOT CHANGE THIS NAME, we rely on it for INSTALL_K3S_PR functionality
+    runs-on: ${{ contains(inputs.arch, 'arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
     timeout-minutes: 20
     env:
       BIN_EXT: ${{ inputs.os == 'windows' && '.exe' || '' }}
-      ARTIFACT_EXT: ${{ inputs.os == 'windows' && '-windows' || (contains(inputs.arch, 'arm') && '-arm64' || '') }}
+      ARCH_EXT: ${{ inputs.os == 'windows' && '-windows' || format('-{0}', inputs.arch) }}
       GOOS: ${{ inputs.os }}
     steps:
     - name: Checkout K3s
       uses: actions/checkout@v4
-      
-    - name: Build K3s binary
+    
+    - name: Set up QEMU
+      if: inputs.arch == 'arm'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: linux/arm/v7
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Determine Git Version Info
+      id: git_vars
       run: |
-        DOCKER_BUILDKIT=1 SKIP_IMAGE=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 GOCOVER=1 GOOS=${{ env.GOOS }} make
-        sha256sum dist/artifacts/k3s${{ env.BIN_EXT }} | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.BIN_EXT }}.sha256sum
+        source ./scripts/git_version.sh
+        {
+          echo "git_tag=${GIT_TAG}"
+          echo "tree_state=${TREE_STATE}"
+          echo "commit=${COMMIT}"
+          echo "dirty=${DIRTY}"
+        } >> "$GITHUB_OUTPUT"
+
+    - name: Login to GitHub Container Registry
+      if: inputs.cache == 'write'
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build K3s Binary Native
+      if: inputs.arch != 'arm'
+      env:
+          DOCKER_BUILD_SUMMARY: false
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./Dockerfile.local
+        target: result
+        # Defined actions like this don't ingest GITHUB_ENV, so use outputs
+        # and manual set the build arguments
+        build-args: |
+          GIT_TAG=${{ steps.git_vars.outputs.git_tag }}
+          TREE_STATE=${{ steps.git_vars.outputs.tree_state }}
+          COMMIT=${{ steps.git_vars.outputs.commit }}
+          DIRTY=${{ steps.git_vars.outputs.dirty }}
+        cache-from: ${{ inputs.cache != '' && format('type=registry,ref=ghcr.io/{0}:cache-{1}', github.repository, inputs.arch) || '' }}
+        cache-to: ${{ inputs.cache == 'write' && format('type=registry,ref=ghcr.io/{0}:cache-{1},mode=max', github.repository, inputs.arch) || '' }}
+        push: false
+        provenance: mode=min
+        outputs: type=local,dest=.
+
+    - name: Build K3s Binary Emulated
+      if: inputs.arch != 'arm64' && inputs.arch != 'amd64'
+      env:
+        PLATFORM: ${{ inputs.arch == 'arm' && 'linux/arm/v7' || format('linux/{0}', inputs.arch) }}
+        DOCKER_BUILD_SUMMARY: false
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./Dockerfile.local
+        target: result
+        build-args: |
+          GIT_TAG=${{ steps.git_vars.outputs.git_tag }}
+          TREE_STATE=${{ steps.git_vars.outputs.tree_state }}
+          COMMIT=${{ steps.git_vars.outputs.commit }}
+          DIRTY=${{ steps.git_vars.outputs.dirty }}
+        push: false
+        provenance: mode=min
+        platforms: ${{ env.PLATFORM }}
+        outputs: type=local,dest=.
+
+    - name: Caculate binary checksum
+      run: |
+        if [ ${{ inputs.arch }} == 'amd64' ]; then
+          sha256sum dist/artifacts/k3s${{ env.BIN_EXT }} | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.BIN_EXT }}.sha256sum
+        elif [ ${{ inputs.arch }} == "arm" ]; then
+          sha256sum dist/artifacts/k3s-armhf | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.ARCH_EXT }}.sha256sum
+        else
+          sha256sum dist/artifacts/k3s${{ env.ARCH_EXT }}${{ env.BIN_EXT }} | sed 's|dist/artifacts/||' > dist/artifacts/k3s${{ env.ARCH_EXT }}${{ env.BIN_EXT }}.sha256sum
+        fi
     
     - name: Build K3s image
       if: inputs.upload-image == true && inputs.os == 'linux'
-      run: make package-image
+      run: ./scripts/package-image
       
     - name: "Save K3s image"
       if: inputs.upload-image == true && inputs.os == 'linux'
@@ -48,5 +132,5 @@ jobs:
     - name: "Upload K3s Artifacts"
       uses: actions/upload-artifact@v4
       with:
-        name: k3s${{ env.ARTIFACT_EXT }}
+        name: k3s${{ env.ARCH_EXT }}
         path: dist/artifacts/k3s*

.github/workflows/e2e.yaml

@@ -29,14 +29,22 @@ permissions:
 
 jobs:
   build:
+    permissions:
+      contents: read
+      packages: write # permissions cannot be conditional, so we need to set this for all jobs
     uses: ./.github/workflows/build-k3s.yaml
     with:
       upload-image: true
+      cache: ${{ github.ref == 'refs/heads/master' && 'write' || 'read' }}
   build-arm64:
     uses: ./.github/workflows/build-k3s.yaml
+    permissions:
+      contents: read
+      packages: write
     with:
-      arch: ubuntu-24.04-arm
+      arch: arm64
       upload-image: true
+      cache: ${{ github.ref == 'refs/heads/master' && 'write' || 'read' }}
   e2e:
     name: "E2E Tests"
     needs: build
@@ -71,10 +79,7 @@ jobs:
       - name: "Vagrant Plugin(s)"
         run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp
       - name: Install Go
-        uses: actions/setup-go@v5
-        with: 
-          go-version-file: 'go.mod'
-          cache: false
+        uses: ./.github/actions/setup-go
       - name: Install Kubectl
         run: |
            curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
@@ -82,7 +87,7 @@ jobs:
       - name: "Download k3s binary"
         uses: actions/download-artifact@v4
         with:
-          name: k3s
+          name: k3s-amd64
           path: ./dist/artifacts
       
       - name: Run ${{ matrix.etest }} Test
@@ -177,17 +182,10 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - name: "Download K3s image (amd64)"
-      if: ${{ matrix.arch == 'amd64' }}
+    - name: "Download K3s image"
       uses: actions/download-artifact@v4
       with:
-        name: k3s
-        path: ./dist/artifacts
-    - name: "Download K3s image (arm64)"
-      if: ${{ matrix.arch == 'arm64' }}
-      uses: actions/download-artifact@v4
-      with:
-        name: k3s-arm64
+        name: k3s-${{ matrix.arch }}
         path: ./dist/artifacts
     - name: Load and set K3s image
       run: |

.github/workflows/install.yaml

@@ -54,7 +54,7 @@ jobs:
       - name: "Download k3s binary"
         uses: actions/download-artifact@v4
         with:
-          name: k3s
+          name: k3s-amd64
           path: tests/install/${{ matrix.vm }}
       - name: "Vagrant Up"
         run: vagrant up --no-tty --no-provision

.github/workflows/integration.yaml

@@ -29,9 +29,13 @@ env:
   
 jobs:
   build:
+    permissions:
+      contents: read
+      packages: read
     uses: ./.github/workflows/build-k3s.yaml
     with:
       os: linux
+      cache: read
   build-windows:
     uses: ./.github/workflows/build-k3s.yaml
     with:
@@ -56,7 +60,7 @@ jobs:
     - name: "Download k3s binary"
       uses: actions/download-artifact@v4
       with:
-        name: k3s
+        name: k3s-amd64
         path: ./dist/artifacts
     - name: Run Integration Tests
       run: | 

.github/workflows/trivy.yaml

@@ -36,8 +36,7 @@ jobs:
 
     - name: Build K3s Image
       run: |
-        make local
-        make package-image
+        make local-image
         make tag-image-latest
 
     - name: Download Rancher's VEX Hub report

Dockerfile.local

@@ -32,10 +32,14 @@ ENV SRC_DIR=/go/src/github.com/k3s-io/k3s
 WORKDIR ${SRC_DIR}/
 
 
-FROM infra AS build
-
-ARG SKIP_VALIDATE
-
+FROM infra AS manifests
+ARG GIT_TAG
+ARG TREE_STATE
+ARG COMMIT
+ARG DIRTY
+ARG GOOS
+ENV NO_DAPPER=true
+# Used by both build and validate stages, better caching if we do this in a separate stage
 COPY ./scripts/ ./scripts
 COPY ./go.mod ./go.sum ./main.go ./
 COPY ./manifests ./manifests
@@ -43,9 +47,21 @@ RUN mkdir -p bin dist
 RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
     ./scripts/download
 
+
+FROM manifests AS validate
+ARG SKIP_VALIDATE
+COPY . .
+RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
+    --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
+    --mount=type=cache,id=lint,target=/root/.cache/golangci-lint \
+    ./scripts/validate
+
+
+FROM manifests AS build
+ARG GOCOVER
+ARG DEBUG
 COPY ./cmd ./cmd
 COPY ./tests ./tests
-COPY ./.git ./.git
 COPY ./pkg ./pkg
 RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
     --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
@@ -58,6 +74,7 @@ RUN --mount=type=cache,id=gomod,target=/go/pkg/mod \
 
 RUN ./scripts/binary_size_check.sh
 
+
 FROM scratch AS result
 ENV SRC_DIR=/go/src/github.com/k3s-io/k3s
 COPY --from=build ${SRC_DIR}/dist /dist

Makefile

@@ -43,8 +43,38 @@ format:
 	gofmt -s -l -w $(GO_FILES)
 	goimports -w $(GO_FILES)
 
-.PHONY: local
-local:
+
+.PHONY: local-validate
+local-validate:
 	DOCKER_BUILDKIT=1 docker build \
-		--build-arg="REPO TAG GITHUB_TOKEN GOLANG GOCOVER DEBUG" \
-		-t k3s-local -f Dockerfile.local --output=. .
+		--build-arg="SKIP_VALIDATE=$(SKIP_VALIDATE)" \
+		--build-arg="DEBUG=$(DEBUG)" \
+		--progress=plain \
+		-f Dockerfile.local --target=validate .
+
+.PHONY: local-binary
+local-binary:
+	@echo "INFO: Building K3s binaries and assets..."
+	. ./scripts/git_version.sh && \
+	DOCKER_BUILDKIT=1 docker build \
+		--build-arg "GIT_TAG=$$GIT_TAG" \
+		--build-arg "TREE_STATE=$$TREE_STATE" \
+		--build-arg "COMMIT=$$COMMIT" \
+		--build-arg "DIRTY=$$DIRTY" \
+		--build-arg="GOCOVER=$(GOCOVER)" \
+		--build-arg="GOOS=$(GOOS)" \
+		--build-arg="DEBUG=$(DEBUG)" \
+		-f Dockerfile.local --target=result --output=. .
+
+.PHONY: local-image
+local-image: local-binary
+	@echo "INFO: Building K3s image..."
+	./scripts/package-image
+
+.PHONY: local-airgap
+local-airgap: 
+	@echo "INFO: Building K3s airgap tarball..."
+	./scripts/package-airgap
+
+.PHONY: local-ci
+local-ci:  local-binary local-image local-airgap

install.sh

@@ -514,8 +514,8 @@ get_pr_artifact_url() {
     # GET request to the GitHub API to retrieve the Build workflow associated with the commit
     run_id=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" "${github_api_url}/commits/${commit_id}/check-runs?check_name=build%20%2F%20Build" | jq -r '[.check_runs | sort_by(.id) | .[].details_url | split("/")[7]] | last')
 
-    # Extract the artifact ID for the "k3s" artifact
-    GITHUB_PR_URL=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" "${github_api_url}/actions/runs/${run_id}/artifacts" | jq -r '.artifacts[] | select(.name == "k3s") | .archive_download_url')
+    # Extract the artifact ID for the "k3s" (old) or "k3s-amd64" (new) artifact
+    GITHUB_PR_URL=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" "${github_api_url}/actions/runs/${run_id}/artifacts" | jq -r '.artifacts[] | select(.name == "k3s" or .name == "k3s-amd64") | .archive_download_url')
 }
 
 # --- download binary from github url ---

install.sh.sha256sum

@@ -1 +1 @@
-9d5fc42bf825d3e8dcc8682c8bac071b1de18019af81f85519ccbe5c919e0896  install.sh
+9ca7930c31179d83bc13de20078fd8ad3e1ee00875b31f39a7e524ca4ef7d9de  install.sh

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21 as base
+FROM alpine:3.21 AS base
 RUN apk add -U ca-certificates zstd tzdata
 COPY build/out/data-linux.tar.zst /
 RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware /image/var/lib/rancher/k3s/data/cni && \
@@ -8,7 +8,7 @@ RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/li
     echo "root:x:0:" > /image/etc/group && \
     cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt
 
-FROM scratch as collect
+FROM scratch AS collect
 ARG DRONE_TAG="dev"
 COPY --from=base /image /
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo

scripts/git_version.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+GIT_TAG=$DRONE_TAG
+TREE_STATE=clean
+COMMIT=$DRONE_COMMIT
+
+if [ -d .git ]; then
+    if [ -z "$GIT_TAG" ]; then
+        GIT_TAG=$(git tag -l --contains HEAD | head -n 1)
+    fi
+    if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+        DIRTY="-dirty"
+        TREE_STATE=dirty
+    fi
+
+    COMMIT=$(git log -n3 --pretty=format:"%H %ae" | grep -v ' drone@localhost$' | cut -f1 -d\  | head -1)
+    if [ -z "${COMMIT}" ]; then
+    COMMIT=$(git rev-parse HEAD || true)
+    fi
+fi
+
+export GIT_TAG
+export TREE_STATE
+export COMMIT
+export DIRTY

scripts/version.sh

@@ -4,23 +4,9 @@ GO=${GO-go}
 ARCH=${ARCH:-$("${GO}" env GOARCH)}
 OS=${OS:-$("${GO}" env GOOS)}
 SUFFIX="-${ARCH}"
-GIT_TAG=$DRONE_TAG
-TREE_STATE=clean
-COMMIT=$DRONE_COMMIT
 
-if [ -d .git ]; then
-    if [ -z "$GIT_TAG" ]; then
-        GIT_TAG=$(git tag -l --contains HEAD | head -n 1)
-    fi
-    if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
-        DIRTY="-dirty"
-        TREE_STATE=dirty
-    fi
-
-    COMMIT=$(git log -n3 --pretty=format:"%H %ae" | grep -v ' drone@localhost$' | cut -f1 -d\  | head -1)
-    if [ -z "${COMMIT}" ]; then
-    COMMIT=$(git rev-parse HEAD || true)
-    fi
+if [ -z "$NO_DAPPER" ]; then
+    . ./scripts/git_version.sh
 fi
 
 get-module-version(){