diff --git a/.github/workflows/build-hcp-image.yml b/.github/workflows/build-hcp-image.yml index 1dc4b4a606..a115638172 100644 --- a/.github/workflows/build-hcp-image.yml +++ b/.github/workflows/build-hcp-image.yml @@ -3,6 +3,8 @@ name: build-hcp-image on: workflow_call: inputs: + branch: + type: string pull-request: type: number create-azure-image: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10915c3e9f..4b28a8da7e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -395,17 +395,20 @@ jobs: hcp-image: if: | needs.setup.outputs.is-ent-branch == 'true' && - needs.setup.outputs.workflow-trigger == 'pull_request' && - ( - contains(fromJSON(needs.setup.outputs.labels), 'hcp/build-image') || - contains(fromJSON(needs.setup.outputs.labels), 'hcp/test') + needs.setup.outputs.workflow-trigger == 'schedule' || + ( needs.setup.outputs.workflow-trigger == 'pull_request' && + ( + contains(fromJSON(needs.setup.outputs.labels), 'hcp/build-image') || + contains(fromJSON(needs.setup.outputs.labels), 'hcp/test') + ) ) needs: - setup - artifacts-ent uses: ./.github/workflows/build-hcp-image.yml with: - pull-request: ${{ github.event.pull_request.number }} + pull-request: ${{ needs.setup.outputs.workflow-trigger == 'pull_request' && github.event.pull_request.number || '' }} + branch: ${{ needs.setup.outputs.workflow-trigger == 'schedule' && 'main' || '' }} create-aws-image: true create-azure-image: false hcp-environment: int @@ -481,6 +484,7 @@ jobs: - hcp-image uses: ./.github/workflows/test-run-enos-scenario-cloud.yml with: + hcp-environment: int product-version: ${{ fromJSON(needs.hcp-image.outputs.image).product_version }} completed-successfully: diff --git a/.github/workflows/test-run-enos-scenario-cloud.yml b/.github/workflows/test-run-enos-scenario-cloud.yml index 22e2b192da..9d91b47ec1 100644 --- a/.github/workflows/test-run-enos-scenario-cloud.yml +++ b/.github/workflows/test-run-enos-scenario-cloud.yml @@ -4,6 +4,8 @@ name: enos-cloud on: workflow_call: inputs: + hcp-environment: + type: string product-version: type: string diff --git a/tools/pipeline/go.mod b/tools/pipeline/go.mod index 9432f9bd30..b05ab96431 100644 --- a/tools/pipeline/go.mod +++ b/tools/pipeline/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/PuerkitoBio/goquery v1.11.0 github.com/avast/retry-go/v4 v4.6.1 - github.com/google/go-github/v74 v74.0.0 + github.com/google/go-github/v81 v81.0.0 github.com/hashicorp/hcl/v2 v2.24.0 github.com/hashicorp/releases-api v0.2.3 github.com/jedib0t/go-pretty/v6 v6.6.8 diff --git a/tools/pipeline/go.sum b/tools/pipeline/go.sum index 6624a90696..dd95898e66 100644 --- a/tools/pipeline/go.sum +++ b/tools/pipeline/go.sum @@ -81,8 +81,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v74 v74.0.0 h1:yZcddTUn8DPbj11GxnMrNiAnXH14gNs559AsUpNpPgM= -github.com/google/go-github/v74 v74.0.0/go.mod h1:ubn/YdyftV80VPSI26nSJvaEsTOnsjrxG3o9kJhcyak= +github.com/google/go-github/v81 v81.0.0 h1:hTLugQRxSLD1Yei18fk4A5eYjOGLUBKAl/VCqOfFkZc= +github.com/google/go-github/v81 v81.0.0/go.mod h1:upyjaybucIbBIuxgJS7YLOZGziyvvJ92WX6WEBNE3sM= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= diff --git a/tools/pipeline/internal/cmd/github.go b/tools/pipeline/internal/cmd/github.go index 694a1ac517..171a5379de 100644 --- a/tools/pipeline/internal/cmd/github.go +++ b/tools/pipeline/internal/cmd/github.go @@ -9,7 +9,7 @@ import ( "os" "path/filepath" - "github.com/google/go-github/v74/github" + "github.com/google/go-github/v81/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/shurcooL/githubv4" "github.com/spf13/cobra" diff --git a/tools/pipeline/internal/cmd/github_find_workflow_artifact.go b/tools/pipeline/internal/cmd/github_find_workflow_artifact.go index 6a784046b1..cdf2d246eb 100644 --- a/tools/pipeline/internal/cmd/github_find_workflow_artifact.go +++ b/tools/pipeline/internal/cmd/github_find_workflow_artifact.go @@ -15,7 +15,7 @@ var findWorkflowArtifact = &github.FindWorkflowArtifactReq{} func newGithubFindWorkflowArtifactCmd() *cobra.Command { findWorkflowArtifactCmd := &cobra.Command{ - Use: "workflow-artifact [--pr 1234 --workflow build --pattern 'vault_[0-9]'", + Use: "workflow-artifact [--pr 1234 | --branch main] [--workflow build --pattern 'vault_[0-9]' ]", Short: "Find an artifact associated with a pull requests workflow run", Long: "Find an artifact associated with a pull requests workflow run", RunE: runFindGithubWorkflowArtifactCmd, @@ -26,6 +26,7 @@ func newGithubFindWorkflowArtifactCmd() *cobra.Command { findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Owner, "owner", "o", "hashicorp", "The Github organization") findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Repo, "repo", "r", "vault", "The Github repository. Private repositories require auth via a GITHUB_TOKEN env var") findWorkflowArtifactCmd.PersistentFlags().IntVarP(&findWorkflowArtifact.PullNumber, "pr", "p", 0, "The pull request to use as the trigger of the workflow") + findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.Branch, "branch", "b", "", "The branch to use as the trigger of the workflow") findWorkflowArtifactCmd.PersistentFlags().StringVarP(&findWorkflowArtifact.WorkflowName, "workflow", "w", "", "The name of the workflow the artifact will be associated with") findWorkflowArtifactCmd.PersistentFlags().BoolVar(&findWorkflowArtifact.WriteToGithubOutput, "github-output", false, "Whether or not to write 'workflow-artifact' to $GITHUB_OUTPUT") diff --git a/tools/pipeline/internal/pkg/changed/checkers_test.go b/tools/pipeline/internal/pkg/changed/checkers_test.go index 6df338e480..a1f6883da9 100644 --- a/tools/pipeline/internal/pkg/changed/checkers_test.go +++ b/tools/pipeline/internal/pkg/changed/checkers_test.go @@ -7,7 +7,7 @@ import ( "context" "testing" - "github.com/google/go-github/v74/github" + "github.com/google/go-github/v81/github" "github.com/stretchr/testify/require" ) diff --git a/tools/pipeline/internal/pkg/changed/file.go b/tools/pipeline/internal/pkg/changed/file.go index c6cb016054..7b9a726ede 100644 --- a/tools/pipeline/internal/pkg/changed/file.go +++ b/tools/pipeline/internal/pkg/changed/file.go @@ -8,7 +8,7 @@ import ( "slices" "strings" - gh "github.com/google/go-github/v74/github" + gh "github.com/google/go-github/v81/github" ) type ( diff --git a/tools/pipeline/internal/pkg/github/add_assignees.go b/tools/pipeline/internal/pkg/github/add_assignees.go index fd82d314cb..580d2a52bb 100644 --- a/tools/pipeline/internal/pkg/github/add_assignees.go +++ b/tools/pipeline/internal/pkg/github/add_assignees.go @@ -8,7 +8,7 @@ import ( "log/slog" "slices" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" slogctx "github.com/veqryn/slog-context" ) diff --git a/tools/pipeline/internal/pkg/github/check_commit_status.go b/tools/pipeline/internal/pkg/github/check_commit_status.go index 36d261eccb..7496e704c5 100644 --- a/tools/pipeline/internal/pkg/github/check_commit_status.go +++ b/tools/pipeline/internal/pkg/github/check_commit_status.go @@ -10,7 +10,7 @@ import ( "slices" "strings" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/jedib0t/go-pretty/v6/table" ) diff --git a/tools/pipeline/internal/pkg/github/check_go_mod_diff_request.go b/tools/pipeline/internal/pkg/github/check_go_mod_diff_request.go index 301e109670..27c56843bb 100644 --- a/tools/pipeline/internal/pkg/github/check_go_mod_diff_request.go +++ b/tools/pipeline/internal/pkg/github/check_go_mod_diff_request.go @@ -11,7 +11,7 @@ import ( "log/slog" "os" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/golang" "github.com/jedib0t/go-pretty/v6/table" diff --git a/tools/pipeline/internal/pkg/github/close_copied_origin_pull_request.go b/tools/pipeline/internal/pkg/github/close_copied_origin_pull_request.go index b8425241f7..4cf9683b9b 100644 --- a/tools/pipeline/internal/pkg/github/close_copied_origin_pull_request.go +++ b/tools/pipeline/internal/pkg/github/close_copied_origin_pull_request.go @@ -12,7 +12,7 @@ import ( "slices" "strings" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/jedib0t/go-pretty/v6/table" "github.com/shurcooL/githubv4" slogctx "github.com/veqryn/slog-context" diff --git a/tools/pipeline/internal/pkg/github/commit.go b/tools/pipeline/internal/pkg/github/commit.go index 23d641b59f..da852d810d 100644 --- a/tools/pipeline/internal/pkg/github/commit.go +++ b/tools/pipeline/internal/pkg/github/commit.go @@ -7,7 +7,7 @@ import ( "context" "log/slog" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" slogctx "github.com/veqryn/slog-context" ) diff --git a/tools/pipeline/internal/pkg/github/copy_pull_request.go b/tools/pipeline/internal/pkg/github/copy_pull_request.go index 919a511d95..cadf68a7f7 100644 --- a/tools/pipeline/internal/pkg/github/copy_pull_request.go +++ b/tools/pipeline/internal/pkg/github/copy_pull_request.go @@ -15,7 +15,7 @@ import ( "slices" "strings" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/jedib0t/go-pretty/v6/table" slogctx "github.com/veqryn/slog-context" diff --git a/tools/pipeline/internal/pkg/github/copy_pull_request_test.go b/tools/pipeline/internal/pkg/github/copy_pull_request_test.go index 558dbde64a..a042ad2141 100644 --- a/tools/pipeline/internal/pkg/github/copy_pull_request_test.go +++ b/tools/pipeline/internal/pkg/github/copy_pull_request_test.go @@ -6,7 +6,7 @@ package github import ( "testing" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/stretchr/testify/require" ) diff --git a/tools/pipeline/internal/pkg/github/create_backport.go b/tools/pipeline/internal/pkg/github/create_backport.go index 9353c03aa3..dba3011cad 100644 --- a/tools/pipeline/internal/pkg/github/create_backport.go +++ b/tools/pipeline/internal/pkg/github/create_backport.go @@ -15,7 +15,7 @@ import ( "slices" "strings" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" diff --git a/tools/pipeline/internal/pkg/github/create_backport_test.go b/tools/pipeline/internal/pkg/github/create_backport_test.go index d021d1d8d6..872cf33527 100644 --- a/tools/pipeline/internal/pkg/github/create_backport_test.go +++ b/tools/pipeline/internal/pkg/github/create_backport_test.go @@ -8,7 +8,7 @@ import ( "errors" "testing" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" "github.com/stretchr/testify/require" diff --git a/tools/pipeline/internal/pkg/github/find_workflow_artifact.go b/tools/pipeline/internal/pkg/github/find_workflow_artifact.go index bdb1f05053..3bdccb90d3 100644 --- a/tools/pipeline/internal/pkg/github/find_workflow_artifact.go +++ b/tools/pipeline/internal/pkg/github/find_workflow_artifact.go @@ -9,11 +9,13 @@ import ( "encoding/json" "errors" "fmt" + "log/slog" "regexp" "slices" - gh "github.com/google/go-github/v74/github" + gh "github.com/google/go-github/v81/github" "github.com/jedib0t/go-pretty/v6/table" + slogctx "github.com/veqryn/slog-context" ) // FindWorkflowArtifactReq is a request to find an artifact associated with a @@ -23,6 +25,7 @@ type FindWorkflowArtifactReq struct { ArtifactPattern string Owner string PullNumber int + Branch string Repo string WorkflowName string WriteToGithubOutput bool @@ -60,21 +63,106 @@ func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (* return nil, fmt.Errorf("getting workflow: %w", err) } - // Get the pull request we're searching - res.PR, err = getPullRequest(ctx, client, r.Owner, r.Repo, r.PullNumber) - if err != nil { - return nil, fmt.Errorf("getting pull request: %w", err) + // Define our matcher. It can either be an exact match from an given name or + // match a given pattern. + byNameOrPattern := func(art *gh.Artifact) bool { + // If we've been given a name locate it by that + if r.ArtifactName != "" { + if art.GetName() == r.ArtifactName { + return true + } + } + // Find it by regex + if r.compiledPattern.MatchString(art.GetName()) { + return true + } + + return false } + if r.PullNumber != 0 { + // We've been configured to search for an artifact in reference to a Pull + // Request. Get the details and then search the branch associated with it. + res.PR, err = getPullRequest(ctx, client, r.Owner, r.Repo, r.PullNumber) + if err != nil { + return nil, fmt.Errorf("getting pull request: %w", err) + } + + res.Artifact, err = findWorkflowArtifact( + ctx, + client, + r.Owner, + r.Repo, + res.Workflow.GetID(), + res.PR.GetHead().GetRef(), + res.PR.GetHead().GetSHA(), + byNameOrPattern, + ) + + return res, err + } + + // We've been configured with a branch. Get the last 5 commits and we'll + // we'll walk back until we hopefully find a workflow with a matching artifact. + // We attempt more than one commit because not all commits to either main + // or release branches are guaranteed to create build artifacts. + + ctx = slogctx.Append(ctx, + slog.String("owner", r.Owner), + slog.String("repo", r.Repo), + slog.String("repo", r.Branch), + ) + slog.Default().DebugContext(ctx, "getting list of commits") + commits, _, err := client.Repositories.ListCommits(ctx, r.Owner, r.Repo, &gh.CommitsListOptions{ + SHA: r.Branch, + ListOptions: gh.ListOptions{PerPage: 5}, + }) + if err != nil { + return nil, fmt.Errorf("getting list of commits: %w", err) + } + + var innerErr error + for _, commit := range commits { + res.Artifact, innerErr = findWorkflowArtifact( + ctx, + client, + r.Owner, + r.Repo, + res.Workflow.GetID(), + r.Branch, + commit.GetSHA(), + byNameOrPattern, + ) + if innerErr != nil { + err = errors.Join(err, innerErr) + continue + } + + return res, nil + } + + return nil, errors.Join(errors.New("unable to find artifact matching given criteria"), err) +} + +func findWorkflowArtifact( + ctx context.Context, + client *gh.Client, + owner string, + repo string, + workflowID int64, + branch string, + sha string, + matcher func(*gh.Artifact) bool, +) (*gh.Artifact, error) { // Get the workflow runs associated with the workflow and the PR opts := &gh.ListWorkflowRunsOptions{ - Branch: res.PR.GetHead().GetRef(), + Branch: branch, ExcludePullRequests: false, - HeadSHA: res.PR.GetHead().GetSHA(), + HeadSHA: sha, ListOptions: gh.ListOptions{PerPage: PerPageMax}, Status: "success", } - runs, err := getWorkflowRuns(ctx, client, r.Owner, r.Repo, res.Workflow.GetID(), opts) + runs, err := getWorkflowRuns(ctx, client, owner, repo, workflowID, opts) if err != nil { return nil, fmt.Errorf("getting workflow runs: %w", err) } @@ -92,26 +180,14 @@ func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (* var artifacts gh.ArtifactList for _, run := range runs { - artifacts, err = getWorkflowRunArtifacts(ctx, client, r.Owner, r.Repo, *run.Run.ID) + artifacts, err = getWorkflowRunArtifacts(ctx, client, owner, repo, *run.Run.ID) if err != nil { return nil, fmt.Errorf("getting artifacts for workflow run %d: %w", *run.Run.ID, err) } for _, art := range artifacts.Artifacts { - // If we've been given a name locate it by that - if r.ArtifactName != "" { - if art.GetName() == r.ArtifactName { - res.Artifact = art - - return res, nil - } - } else { - // Find it by regex - if r.compiledPattern.MatchString(art.GetName()) { - res.Artifact = art - - return res, nil - } + if matcher(art) { + return art, nil } } } @@ -134,8 +210,8 @@ func (r *FindWorkflowArtifactReq) validate() error { return errors.New("no github repository has been provided") } - if r.PullNumber == 0 { - return errors.New("no github pull request number has been provided") + if r.PullNumber == 0 && r.Branch == "" { + return errors.New("no github pull request number or branch has been provided") } if r.WorkflowName == "" { diff --git a/tools/pipeline/internal/pkg/github/issue.go b/tools/pipeline/internal/pkg/github/issue.go index 8547eabcb3..e817c82c00 100644 --- a/tools/pipeline/internal/pkg/github/issue.go +++ b/tools/pipeline/internal/pkg/github/issue.go @@ -7,7 +7,7 @@ import ( "context" "log/slog" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" slogctx "github.com/veqryn/slog-context" ) diff --git a/tools/pipeline/internal/pkg/github/list_changed_files.go b/tools/pipeline/internal/pkg/github/list_changed_files.go index e2a76f2f35..d64ccdd7c8 100644 --- a/tools/pipeline/internal/pkg/github/list_changed_files.go +++ b/tools/pipeline/internal/pkg/github/list_changed_files.go @@ -10,7 +10,7 @@ import ( "fmt" "strings" - gh "github.com/google/go-github/v74/github" + gh "github.com/google/go-github/v81/github" "github.com/hashicorp/vault/tools/pipeline/internal/pkg/changed" "github.com/jedib0t/go-pretty/v6/table" ) diff --git a/tools/pipeline/internal/pkg/github/list_commit_statuses.go b/tools/pipeline/internal/pkg/github/list_commit_statuses.go index e0269e28d5..6e91399e0f 100644 --- a/tools/pipeline/internal/pkg/github/list_commit_statuses.go +++ b/tools/pipeline/internal/pkg/github/list_commit_statuses.go @@ -8,7 +8,7 @@ import ( "errors" "fmt" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/jedib0t/go-pretty/v6/table" ) diff --git a/tools/pipeline/internal/pkg/github/list_workflow_runs.go b/tools/pipeline/internal/pkg/github/list_workflow_runs.go index 523b2ad6c1..e4af16a646 100644 --- a/tools/pipeline/internal/pkg/github/list_workflow_runs.go +++ b/tools/pipeline/internal/pkg/github/list_workflow_runs.go @@ -10,7 +10,7 @@ import ( "net/http" "sync" - gh "github.com/google/go-github/v74/github" + gh "github.com/google/go-github/v81/github" ) // PerPageMax is the maximum number of entities to request for enpoints that diff --git a/tools/pipeline/internal/pkg/github/pull_request.go b/tools/pipeline/internal/pkg/github/pull_request.go index a640709cf7..df1e274c41 100644 --- a/tools/pipeline/internal/pkg/github/pull_request.go +++ b/tools/pipeline/internal/pkg/github/pull_request.go @@ -8,7 +8,7 @@ import ( "fmt" "log/slog" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/shurcooL/githubv4" slogctx "github.com/veqryn/slog-context" ) diff --git a/tools/pipeline/internal/pkg/github/sync_branch_request.go b/tools/pipeline/internal/pkg/github/sync_branch_request.go index 536834db54..2f9cd074c7 100644 --- a/tools/pipeline/internal/pkg/github/sync_branch_request.go +++ b/tools/pipeline/internal/pkg/github/sync_branch_request.go @@ -12,7 +12,7 @@ import ( "os" "path/filepath" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" libgit "github.com/hashicorp/vault/tools/pipeline/internal/pkg/git" "github.com/jedib0t/go-pretty/v6/table" slogctx "github.com/veqryn/slog-context" diff --git a/tools/pipeline/internal/pkg/github/templates_test.go b/tools/pipeline/internal/pkg/github/templates_test.go index 6732ad7ba6..83d61bf674 100644 --- a/tools/pipeline/internal/pkg/github/templates_test.go +++ b/tools/pipeline/internal/pkg/github/templates_test.go @@ -8,7 +8,7 @@ import ( "io" "testing" - libgithub "github.com/google/go-github/v74/github" + libgithub "github.com/google/go-github/v81/github" "github.com/stretchr/testify/require" ) diff --git a/tools/pipeline/internal/pkg/github/workflows.go b/tools/pipeline/internal/pkg/github/workflows.go index 20cde452ce..0bc86120ea 100644 --- a/tools/pipeline/internal/pkg/github/workflows.go +++ b/tools/pipeline/internal/pkg/github/workflows.go @@ -8,7 +8,7 @@ import ( "fmt" "log/slog" - gh "github.com/google/go-github/v74/github" + gh "github.com/google/go-github/v81/github" slogctx "github.com/veqryn/slog-context" )