From 979f73bf7d3cd43a87c812c43f68c814ee55070c Mon Sep 17 00:00:00 2001 From: zylxjtu Date: Thu, 11 Dec 2025 18:24:30 +0000 Subject: [PATCH] Add the fake registry server functionality to agnhost windows there are recent changes in k8s e2e test for image pull test to switch to use fake registry server, https://github.com/kubernetes/kubernetes/pull/133272 https://github.com/kubernetes/kubernetes/pull/134453 Unfortunately, this does not take into account of windows node. so the corresonding test on windows node strat to break The try to address this issue, by (1): update the agnhost windows image to include the fake registery server functionality as well (2): update the (common) pull image test to include windows node, will require (a): deploy the agnhost faker register server as a HPC pod (b): update the comanndline/options for the windows container specific --- test/e2e/framework/registry/registry.go | 4 ++ test/images/agnhost/Dockerfile_windows | 20 ++++++++- test/images/agnhost/VERSION | 2 +- .../fakeregistryserver/prepare_registry.sh | 45 ++++++++++++++----- .../fakeregistryserver/registryserver.go | 30 +++++++++++-- .../fakeregistryserver/registryserver_test.go | 44 +++++++++++++----- 6 files changed, 119 insertions(+), 26 deletions(-) diff --git a/test/e2e/framework/registry/registry.go b/test/e2e/framework/registry/registry.go index 139d61aec3c..d4839191799 100644 --- a/test/e2e/framework/registry/registry.go +++ b/test/e2e/framework/registry/registry.go @@ -127,6 +127,10 @@ func podManifest(podTestLabel string) (*v1.Pod, error) { }, } pod.Spec.Containers[0].SecurityContext.RunAsUser = ptr.To[int64](5123) + pod.Spec.Containers[0].SecurityContext.WindowsOptions = &v1.WindowsSecurityContextOptions{ + RunAsUserName: ptr.To("NT AUTHORITY\\SYSTEM"), + HostProcess: ptr.To(true), + } // setting HostNetwork to true so that the registry is accessible on localhost: // and we don't have to deal with any CNI quirks. pod.Spec.HostNetwork = true diff --git a/test/images/agnhost/Dockerfile_windows b/test/images/agnhost/Dockerfile_windows index f2b50fb0700..b7f5dc97571 100644 --- a/test/images/agnhost/Dockerfile_windows +++ b/test/images/agnhost/Dockerfile_windows @@ -15,9 +15,23 @@ ARG BASEIMAGE ARG REGISTRY ARG OS_VERSION +ARG GOLANG_VERSION + +FROM --platform=linux/$BUILDARCH golang:$GOLANG_VERSION AS prepregistry + +RUN go install github.com/google/go-containerregistry/cmd/crane@latest && \ + apt-get update && apt-get install -y jq + +COPY fakeregistryserver/prepare_registry.sh /prepare_registry.sh +COPY fakeregistryserver/images.txt /images.txt + +RUN chmod +x /prepare_registry.sh + +# run the script during the build to create the artifact inside the image +RUN /prepare_registry.sh # We're using a Linux image to unpack the archives, then we're copying them over to Windows. -FROM --platform=linux/amd64 alpine:3.21 as prep +FROM --platform=linux/$BUILDARCH alpine:3.21 as prep ADD https://github.com/coredns/coredns/releases/download/v1.5.0/coredns_1.5.0_windows_amd64.tgz /coredns.tgz ADD https://iperf.fr/download/windows/iperf-2.0.9-win64.zip /iperf.zip @@ -32,6 +46,8 @@ RUN tar -xzvf /coredns.tgz &&\ mv coreutils-8.31-28-windows-64bit wincoreutils &&\ mkdir /uploads +# TODO: Change to linux/$BUILDARCH when windows-servercore-cache has arm64 variant +# Keep linux/amd64 hardcoded: windows-servercore-cache only has amd64 variant (no arm64 available) FROM --platform=linux/amd64 $REGISTRY/windows-servercore-cache:1.0-linux-amd64-$OS_VERSION as servercore-helper FROM $BASEIMAGE @@ -48,6 +64,8 @@ COPY --from=servercore-helper /Windows/System32/netapi32.dll /Windows/System32/n COPY --from=prep /coredns.exe /coredns.exe COPY --from=prep /iperf /iperf COPY --from=prep /wincoreutils/sync.exe /bin/sync.exe +# Copy the fake registry data from the preparer stage +COPY --from=prepregistry /registry /var/registry # NOTE(claudiub): docker buildx sets the PATH env variable to a Linux-like PATH, which is not desirable. ENV PATH="C:\dig\;C:\bin\;C:\curl\;C:\Windows\system32;C:\Windows;C:\Program Files\PowerShell;" diff --git a/test/images/agnhost/VERSION b/test/images/agnhost/VERSION index 578dac52eb2..aa97c2f1d7f 100644 --- a/test/images/agnhost/VERSION +++ b/test/images/agnhost/VERSION @@ -1 +1 @@ -2.62.1 +2.62.2 diff --git a/test/images/agnhost/fakeregistryserver/prepare_registry.sh b/test/images/agnhost/fakeregistryserver/prepare_registry.sh index 1912ff2c0b3..71b42aa922d 100755 --- a/test/images/agnhost/fakeregistryserver/prepare_registry.sh +++ b/test/images/agnhost/fakeregistryserver/prepare_registry.sh @@ -24,6 +24,18 @@ readonly REGISTRY_DIR="/registry" # This script prepares a directory with container images to be used as a fake registry, # then creates a tarball of that directory inside the container. +# digest_to_path converts a digest to a nested directory path following the Docker +# registry storage layout. For example, "sha256:abc123..." becomes +# "/sha256/ab/abc123.../data". +digest_to_path() { + local base_dir="$1" + local digest="$2" + local algo="${digest%%:*}" # e.g., "sha256" + local hash="${digest#*:}" # e.g., "abc123..." + local prefix="${hash:0:2}" # first 2 chars + echo "$base_dir/$algo/$prefix/$hash/data" +} + # function to download an image manifest and its blobs to create a fake registry layout. prepare_image() { local image_name="$1" @@ -42,33 +54,44 @@ prepare_image() { crane manifest "$REGISTRY_URL/$image_name:$tag" | jq '.manifests |= map(select(.platform.os != "windows"))' > "$tmp_manifest_path" echo "Saved manifest list to $tmp_manifest_path" - local manifest_digest - manifest_digest="sha256:$(sha256sum < "$tmp_manifest_path" | awk '{print $1}')" - mv "$tmp_manifest_path" "$image_dir/manifests/$manifest_digest" - echo "Saved manifest list to $image_dir/manifests/$manifest_digest" + local manifest_hash manifest_digest manifest_path + manifest_hash="$(sha256sum < "$tmp_manifest_path" | awk '{print $1}')" + manifest_digest="sha256:$manifest_hash" + manifest_path="$(digest_to_path "$image_dir/manifests" "$manifest_digest")" + mkdir -p "$(dirname "$manifest_path")" + mv "$tmp_manifest_path" "$manifest_path" + echo "Saved manifest list to $manifest_path" # the file named after the tag now contains only the digest, acting as a redirect pointer echo "$manifest_digest" > "$image_dir/manifests/${internal_tag}" echo "Created tag file ${internal_tag} pointing to digest $manifest_digest" echo "Parsing manifest list and downloading individual manifests and blobs..." - - jq -r '.manifests[].digest' < "$image_dir/manifests/$manifest_digest" | while read -r individual_manifest_digest; do + + jq -r '.manifests[].digest' < "$manifest_path" | while read -r individual_manifest_digest; do echo " Downloading manifest $individual_manifest_digest..." - local individual_manifest_path="$image_dir/manifests/$individual_manifest_digest" + local individual_manifest_path + individual_manifest_path="$(digest_to_path "$image_dir/manifests" "$individual_manifest_digest")" + mkdir -p "$(dirname "$individual_manifest_path")" crane manifest "$REGISTRY_URL/$image_name@$individual_manifest_digest" > "$individual_manifest_path" echo " Saved manifest to $individual_manifest_path" local config_digest config_digest=$(jq -r '.config.digest' < "$individual_manifest_path") echo " Downloading config blob $config_digest..." - crane blob "$REGISTRY_URL/$image_name@$config_digest" > "$image_dir/blobs/$config_digest" - echo " Saved config blob to $image_dir/blobs/$config_digest" + local config_path + config_path="$(digest_to_path "$image_dir/blobs" "$config_digest")" + mkdir -p "$(dirname "$config_path")" + crane blob "$REGISTRY_URL/$image_name@$config_digest" > "$config_path" + echo " Saved config blob to $config_path" jq -r '.layers[].digest' < "$individual_manifest_path" | while read -r layer_digest; do echo " Downloading layer blob $layer_digest..." - crane blob "$REGISTRY_URL/$image_name@$layer_digest" > "$image_dir/blobs/$layer_digest" - echo " Saved layer blob to $image_dir/blobs/$layer_digest" + local layer_path + layer_path="$(digest_to_path "$image_dir/blobs" "$layer_digest")" + mkdir -p "$(dirname "$layer_path")" + crane blob "$REGISTRY_URL/$image_name@$layer_digest" > "$layer_path" + echo " Saved layer blob to $layer_path" done done diff --git a/test/images/agnhost/fakeregistryserver/registryserver.go b/test/images/agnhost/fakeregistryserver/registryserver.go index 06b4211b8ed..805f30d88c6 100644 --- a/test/images/agnhost/fakeregistryserver/registryserver.go +++ b/test/images/agnhost/fakeregistryserver/registryserver.go @@ -22,6 +22,7 @@ import ( "log" "net/http" "os" + "path/filepath" "strings" "github.com/spf13/cobra" @@ -52,6 +53,8 @@ var CmdFakeRegistryServer = &cobra.Command{ } func main(cmd *cobra.Command, args []string) { + log.Printf("Registry server starting with registry directory: %s", registryDir) + registryMux := NewRegistryServerMux(private) addr := fmt.Sprintf(":%d", port) @@ -86,9 +89,23 @@ func auth(h http.Handler) http.Handler { }) } +// digestToPath converts a digest to a nested directory path following the Docker +// registry storage layout. For example, "sha256:abc123..." becomes +// "/sha256/ab/abc123.../data". This avoids filesystem issues with colons +// in filenames on Windows and matches the standard Docker registry layout. +func digestToPath(baseDir, digest string) string { + parts := strings.SplitN(digest, ":", 2) + if len(parts) != 2 || len(parts[1]) < 2 { + return filepath.Join(baseDir, digest) + } + algo := parts[0] // e.g., "sha256" + hash := parts[1] // e.g., "abc123..." + return filepath.Join(baseDir, algo, hash[:2], hash, "data") +} + // handleBlobs serves blob requests func handleBlobs(w http.ResponseWriter, r *http.Request, imageName, identifier string) { - filePath := fmt.Sprintf("%s/%s/blobs/%s", registryDir, imageName, identifier) + filePath := digestToPath(filepath.Join(registryDir, imageName, "blobs"), identifier) w.Header().Set("Content-Type", "application/octet-stream") log.Printf("Serving blob: %s", filePath) http.ServeFile(w, r, filePath) @@ -98,12 +115,16 @@ func handleBlobs(w http.ResponseWriter, r *http.Request, imageName, identifier s // based on the manifest's mediaType field. If the identifier is a tag, it // reads the digest from the tag file and issues a redirect. func handleManifests(w http.ResponseWriter, r *http.Request, imageName, identifier string) { - filePath := fmt.Sprintf("%s/%s/manifests/%s", registryDir, imageName, identifier) + manifestsDir := filepath.Join(registryDir, imageName, "manifests") // if the identifier is not a digest, assume it's a tag and perform a redirect. if !strings.HasPrefix(identifier, "sha256:") { - digest, err := os.ReadFile(filePath) + // Tags are stored as flat files (not nested) + tagFilePath := filepath.Join(manifestsDir, identifier) + log.Printf("Looking for tag file: %s", tagFilePath) + digest, err := os.ReadFile(tagFilePath) if err != nil { + log.Printf("Tag file not found: %s, error: %v", tagFilePath, err) http.NotFound(w, r) return } @@ -113,8 +134,11 @@ func handleManifests(w http.ResponseWriter, r *http.Request, imageName, identifi return } + filePath := digestToPath(manifestsDir, identifier) + log.Printf("Serving manifest: %s", filePath) manifestContent, err := os.ReadFile(filePath) if err != nil { + log.Printf("Manifest file not found: %s, error: %v", filePath, err) http.NotFound(w, r) return } diff --git a/test/images/agnhost/fakeregistryserver/registryserver_test.go b/test/images/agnhost/fakeregistryserver/registryserver_test.go index 890cbb03369..7e2e5d18d3b 100644 --- a/test/images/agnhost/fakeregistryserver/registryserver_test.go +++ b/test/images/agnhost/fakeregistryserver/registryserver_test.go @@ -66,25 +66,27 @@ func setupTestRegistry(t *testing.T) (string, func() error) { manifestsDir := filepath.Join(tempDir, testImageName, "manifests") blobsDir := filepath.Join(tempDir, testImageName, "blobs") - if err := os.MkdirAll(manifestsDir, 0755); err != nil { + + // write the manifest file using nested path + manifestPath := digestToPath(manifestsDir, testManifestDigest) + if err := os.MkdirAll(filepath.Dir(manifestPath), 0755); err != nil { t.Fatalf("Failed to create manifests dir: %v", err) } - if err := os.MkdirAll(blobsDir, 0755); err != nil { - t.Fatalf("Failed to create blobs dir: %v", err) - } - - // write the manifest file - if err := os.WriteFile(filepath.Join(manifestsDir, testManifestDigest), []byte(testManifestContent), 0644); err != nil { + if err := os.WriteFile(manifestPath, []byte(testManifestContent), 0644); err != nil { t.Fatalf("Failed to write manifest file: %v", err) } - // write the tag file + // write the tag file (tags are still flat files) if err := os.WriteFile(filepath.Join(manifestsDir, testTag), []byte(testManifestDigest), 0644); err != nil { t.Fatalf("Failed to write tag file: %v", err) } - // write the blob file - if err := os.WriteFile(filepath.Join(blobsDir, testBlobDigest), []byte(testBlobContent), 0644); err != nil { + // write the blob file using nested path + blobPath := digestToPath(blobsDir, testBlobDigest) + if err := os.MkdirAll(filepath.Dir(blobPath), 0755); err != nil { + t.Fatalf("Failed to create blobs dir: %v", err) + } + if err := os.WriteFile(blobPath, []byte(testBlobContent), 0644); err != nil { t.Fatalf("Failed to write blob file: %v", err) } @@ -95,6 +97,28 @@ func setupTestRegistry(t *testing.T) (string, func() error) { return tempDir, cleanup } +func TestDigestToPath(t *testing.T) { + tests := []struct { + name string + baseDir string + digest string + expected string + }{ + {"valid sha256", "/base", "sha256:abc123", "/base/sha256/ab/abc123/data"}, + {"short hash", "/base", "sha256:a", "/base/sha256:a"}, // fallback + {"no colon", "/base", "invalid", "/base/invalid"}, + {"empty hash", "/base", "sha256:", "/base/sha256:"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := digestToPath(tt.baseDir, tt.digest) + if got != tt.expected { + t.Errorf("digestToPath() = %v, want %v", got, tt.expected) + } + }) + } +} + func TestRegistryServer(t *testing.T) { tempDir, cleanup := setupTestRegistry(t) defer func() {