From 2e156b58285bfcad2aa871c35d7c290c91dde0ae Mon Sep 17 00:00:00 2001 From: subencheng Date: Fri, 11 Sep 2026 15:00:56 -0700 Subject: [PATCH] Build linux release binaries on a dedicated ubuntu runner **Why** The release critical path is the macOS binaries job, and almost all of it is one step. On baton-okta v0.5.36 (12m07s end to end), goreleaser-binaries took 9m15s, of which "Run GoReleaser" was 447s and the compile phase alone was 6m45s: 18:15:20 3 builds start (linux/amd64, linux/arm64, darwin/amd64) 18:20:51 darwin/arm64 finally starts <- waited 5m31s for a CPU slot 18:22:05 compile phase done (6m45s) macos-latest has 3 vCPUs and GoReleaser builds one target per core, so the fourth target sat idle for over five minutes. Notarization is not the bottleneck: the gon hooks took 9s. Only the darwin targets need codesign/notarize, so the linux builds were occupying the scarcest runner in the matrix for no reason. **What this changes** - Adds a goreleaser-linux job on ubuntu-latest that builds linux amd64/arm64, signs, generates SBOMs and provenance, uploads to S3, and emits a manifest fragment plus its checksums. It mirrors the existing goreleaser-windows job and runs with --skip=publish; the darwin job still owns the GitHub release. - Narrows the binaries template to the darwin targets and adds a linux template. - merge-manifests takes -linux-manifest and merges its assets map, skipping the partial checksums entry that publish-release-manifest recomputes. - publish-release-manifest concatenates the linux checksums into the unified file. Two contracts a single GoReleaser run used to cover implicitly, now handled explicitly so they do not silently regress: - The Homebrew formula. GoReleaser can only build formula entries from artifacts in its own run, so a darwin-only run would drop the tap's on_linux blocks. The formula is now rendered by cmd/generate-brew-formula from the merged manifest, where every platform is present, and pushed to the tap by publish-release-manifest. Rendering it against baton-okta v0.5.36 reproduces the published formula exactly apart from the header comment and the CDN URLs. - The GitHub release contents. GoReleaser now attaches only the darwin archives and a darwin-only checksums file, so publish-release-manifest re-attaches the linux archives and clobbers the checksums file with the unified one. The release keeps the same five assets it has today. Formula download URLs move from GitHub release assets to dist.conductorone.com, which is already the canonical distribution channel and what the registry records. **Validation** - go build ./... , go vet ./... , go test ./cmd/... (new tests cover formula class naming, incomplete-asset skipping, and per-OS block rendering). - scripts/test-release-config-templates.py, extended to assert the binaries and linux templates build disjoint GOOS sets and that no template declares brews. - scripts/test-release-workflow-tag-pin.sh, extended to cover the new GoReleaser invocation; the invariant was verified directly since yq was unavailable locally. - scripts/test-normalize-release-options.sh, scripts/test-derive-iam-role-name.sh. - Parsed .github/workflows/release.yaml with PyYAML. Not validated: the workflow has not been executed. Job-level behaviour needs a real release, so this should go out on a prerelease tag against one connector first. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release.yaml | 405 +++++++++++++++++- cmd/generate-brew-formula/main.go | 208 +++++++++ cmd/generate-brew-formula/main_test.go | 114 +++++ cmd/merge-manifests/main.go | 42 ++ scripts/test-release-config-templates.py | 22 +- scripts/test-release-workflow-tag-pin.sh | 1 + .../.goreleaser-binaries-template.yaml.tmpl | 49 +-- .../.goreleaser-linux-template.yaml.tmpl | 60 +++ 8 files changed, 854 insertions(+), 47 deletions(-) create mode 100644 cmd/generate-brew-formula/main.go create mode 100644 cmd/generate-brew-formula/main_test.go create mode 100644 templates/.goreleaser-linux-template.yaml.tmpl diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 73b53d9..7f65950 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -301,6 +301,8 @@ jobs: GOOS="$goos" govulncheck ./... done + # Builds the darwin targets only; linux moved to goreleaser-linux and Windows to + # goreleaser-windows. This job still owns the GitHub release and the changelog. goreleaser-binaries: needs: [determine-workflows-ref, release-preflight] runs-on: macos-latest @@ -382,8 +384,6 @@ jobs: env: REPO_NAME: ${{ github.event.repository.name }} GO_MAIN_PACKAGE: ${{ needs.determine-workflows-ref.outputs.go_main_package }} - BREW_TAP: ${{ needs.determine-workflows-ref.outputs.brew_tap }} - BREW_SKIP_UPLOAD: ${{ inputs.brew != true }} # For provenance predicate template WORKFLOWS_REF: ${{ needs.determine-workflows-ref.outputs.ref }} RELEASE_TAG: ${{ inputs.tag }} @@ -617,6 +617,285 @@ jobs: echo "${DELIM}" } >> "$GITHUB_OUTPUT" + # Linux binaries build on their own ubuntu runner so they compile in parallel with the + # macOS job rather than competing for its 3 vCPUs. The macOS runner is reserved for the + # darwin targets, which are the only ones that need codesign/notarization. + goreleaser-linux: + needs: [determine-workflows-ref, release-preflight] + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # <-- needed for cosign keyless (OIDC) + outputs: + linux_manifest: ${{ steps.generate-linux-manifest.outputs.linux_manifest }} + linux_checksums: ${{ steps.output-checksums.outputs.checksums }} + steps: + - name: Checkout caller repo + uses: actions/checkout@v5 + with: + path: _caller + repository: ${{ github.event.repository.full_name }} + ref: refs/tags/${{ inputs.tag }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify caller checkout matches release tag + working-directory: _caller + shell: bash + env: + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + tag_commit="$(git rev-list -n 1 "refs/tags/$RELEASE_TAG")" + head_commit="$(git rev-parse HEAD)" + if [ "$head_commit" != "$tag_commit" ]; then + echo "::error::Checked out $head_commit but refs/tags/$RELEASE_TAG resolves to $tag_commit" + exit 1 + fi + echo "Verified $RELEASE_TAG at $head_commit" + + - name: Checkout connector workflows + uses: actions/checkout@v5 + with: + path: _workflows + repository: ConductorOne/github-workflows + ref: ${{ needs.determine-workflows-ref.outputs.ref }} + persist-credentials: false + + - name: Derive AWS role names + id: role-names + working-directory: _workflows + shell: bash + env: + REPO_OWNER: ${{ github.event.repository.owner.login }} + REPO_NAME: ${{ github.event.repository.name }} + run: | + bash ./scripts/derive-iam-role-name.sh \ + --prefix GHA-Artifacts- \ + --suffix "${REPO_OWNER}-${REPO_NAME}" \ + --output-name gha_artifacts_role_name >> "$GITHUB_OUTPUT" + + - name: Set up Go for caller + uses: actions/setup-go@v6 + with: + go-version-file: "_caller/go.mod" + cache: false + + - name: Calculate S3 directory + id: s3-directory + shell: bash + run: | + ORG="${{ github.event.repository.owner.login }}" + REPO="${{ github.event.repository.name }}" + STORAGE_NAME="${{ inputs.release_storage_name }}" + TAG="${{ inputs.tag }}" + if [ -z "$STORAGE_NAME" ]; then + STORAGE_NAME="$REPO" + fi + echo "S3_DIRECTORY=releases/$ORG/$STORAGE_NAME/$TAG" >> "$GITHUB_OUTPUT" + + - name: Generate configs for linux + working-directory: _workflows + env: + REPO_NAME: ${{ github.event.repository.name }} + GO_MAIN_PACKAGE: ${{ needs.determine-workflows-ref.outputs.go_main_package }} + # For provenance predicate template + WORKFLOWS_REF: ${{ needs.determine-workflows-ref.outputs.ref }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + mkdir -p "${GENERATED_DIR}" + export BUILD_STARTED_ON=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + envsubst < templates/.goreleaser-linux-template.yaml.tmpl | tee "${GENERATED_DIR}/.goreleaser.linux.yaml" + envsubst < templates/.slsa-provenance-predicate-template.json.tmpl | tee "${GENERATED_DIR}/predicate.json" + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Download syft + uses: anchore/sbom-action/download-syft@v0 + + - name: Configure AWS credentials via OIDC + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: arn:aws:iam::025044153841:role/${{ steps.role-names.outputs.gha_artifacts_role_name }} + aws-region: us-west-2 + + # --skip=publish: the darwin job owns the GitHub release and the changelog. + - name: Run GoReleaser for linux + uses: goreleaser/goreleaser-action@v6 + with: + workdir: _caller + version: "~> v2.13" + args: release --clean --skip=publish --config ../_workflows/_generated/.goreleaser.linux.yaml + env: + GITHUB_TOKEN: ${{ secrets.RELENG_GITHUB_TOKEN }} + GORELEASER_CURRENT_TAG: ${{ inputs.tag }} + + - name: Verify binary module version + if: inputs.verify_module_version == true + working-directory: _caller + shell: bash + env: + RELEASE_TAG: ${{ inputs.tag }} + REPOSITORY_NAME: ${{ github.event.repository.name }} + run: | + set -euo pipefail + binary="$(find dist -type f -name "$REPOSITORY_NAME" -print -quit)" + if [ -z "$binary" ]; then + echo "::error::No generated $REPOSITORY_NAME binary found for module version verification" + exit 1 + fi + module_version="$(go version -m "$binary" | awk '$1 == "mod" { print $3; exit }')" + if [ "$module_version" != "$RELEASE_TAG" ]; then + echo "::error::Generated binary module version is '$module_version', expected '$RELEASE_TAG'" + exit 1 + fi + + - name: Generate SLSA provenance for archives + working-directory: _workflows + env: + CALLER_DIST: ../_caller/dist + shell: bash + run: | + set -euo pipefail + + PROVENANCE_COUNT=0 + + for artifact in "${CALLER_DIST}"/*.tar.gz; do + [ -f "$artifact" ] || continue + [[ "$artifact" == *checksums* ]] && continue + + BASENAME=$(basename "$artifact") + echo "Generating provenance for: $BASENAME" + cosign attest-blob \ + --yes \ + --predicate "${GENERATED_DIR}/predicate.json" \ + --type slsaprovenance1 \ + --bundle "${CALLER_DIST}/${BASENAME}.provenance.sigstore.json" \ + "$artifact" > /dev/null + echo "✅ Created ${BASENAME}.provenance.sigstore.json" + ((PROVENANCE_COUNT++)) || true + done + + echo "Generated provenance bundles: ${PROVENANCE_COUNT}" + if [ "$PROVENANCE_COUNT" -eq 0 ]; then + echo "::error::No provenance bundles were generated - this indicates a build problem" + exit 1 + fi + ls "${CALLER_DIST}"/*.provenance.sigstore.json + + - name: Sign SBOMs as attestation bundles + working-directory: _workflows + env: + CALLER_DIST: ../_caller/dist + shell: bash + run: | + set -euo pipefail + + SIGNED_COUNT=0 + + # Require every linux release archive to have an SPDX SBOM. + for artifact in "${CALLER_DIST}"/*.tar.gz; do + [ -f "$artifact" ] || continue + [[ "$artifact" == *checksums* ]] && continue + + SBOM="${artifact}.sbom.json" + if [ ! -f "$SBOM" ]; then + echo "::error::Missing SBOM for artifact: $(basename "$artifact") (expected: $SBOM)" + exit 1 + fi + + echo "Signing SBOM for: $(basename "$artifact")" + cosign attest-blob \ + --yes \ + --predicate "$SBOM" \ + --type https://spdx.dev/Document \ + --bundle "${artifact}.sbom.sigstore.json" \ + "$artifact" > /dev/null + echo "✅ Created $(basename "$artifact").sbom.sigstore.json" + ((SIGNED_COUNT++)) || true + done + + echo "Generated SBOM bundles: ${SIGNED_COUNT}" + if [ "$SIGNED_COUNT" -eq 0 ]; then + echo "::error::No linux SBOM bundles were generated - this indicates a build problem" + exit 1 + fi + ls "${CALLER_DIST}"/*.sbom.sigstore.json + + - name: Upload linux release artifacts to S3 + working-directory: _workflows + env: + S3_BUCKET: ${{ env.S3_BUCKET }} + S3_DIRECTORY: ${{ steps.s3-directory.outputs.S3_DIRECTORY }} + shell: bash + run: | + set -euo pipefail + ./scripts/upload-release-artifacts.sh \ + --bucket "$S3_BUCKET" \ + --directory "$S3_DIRECTORY" \ + --base-dir "../_caller/dist" + + - name: Set up Go for workflows + uses: actions/setup-go@v6 + with: + go-version-file: "_workflows/go.mod" + cache: false + + # Only the assets map of this manifest is merged; the top-level fields (semver, + # released_at, hrefs) come from the binaries manifest, so released-at is not passed. + - name: Generate linux manifest + id: generate-linux-manifest + working-directory: _workflows + env: + CALLER_DIST: ../_caller/dist + run: | + MANIFEST_JSON=$(go run ./cmd/generate-manifest \ + -asset-dir "${CALLER_DIST}" \ + -repo-name "${{ github.event.repository.name }}" \ + -org-name "${{ github.event.repository.owner.login }}" \ + -tag "${{ inputs.tag }}" \ + -base-url "${{ env.CDN_BASE_URL }}/${{ steps.s3-directory.outputs.S3_DIRECTORY }}") + + echo "$MANIFEST_JSON" + + { + echo "linux_manifest<> "$GITHUB_OUTPUT" + + # The darwin job creates the GitHub release, so this job cannot attach to it yet. + # Hand the archives to publish-release-manifest, which runs after both. + - name: Stage linux archives for the GitHub release + uses: actions/upload-artifact@v4 + with: + name: linux-release-archives + path: _caller/dist/*.tar.gz + if-no-files-found: error + retention-days: 1 + + - name: Output checksums for merging + id: output-checksums + working-directory: _caller + run: | + CHECKSUMS_FILE=$(ls dist/*checksums*.txt 2>/dev/null | head -1) + if [ -z "$CHECKSUMS_FILE" ]; then + echo "::error::No checksums file found" + exit 1 + fi + + echo "Found checksums file: $CHECKSUMS_FILE" + + # Use randomized delimiter to prevent injection via filenames containing "EOF" + DELIM="CHECKSUMS_$(openssl rand -hex 8)" + { + echo "checksums<<${DELIM}" + cat "$CHECKSUMS_FILE" + echo "${DELIM}" + } >> "$GITHUB_OUTPUT" + goreleaser-windows: if: inputs.msi == true needs: [determine-workflows-ref, release-preflight] @@ -1216,11 +1495,11 @@ jobs: publish-release-manifest: # Release manifest publication: manifest + checksums + S3 upload. - # Require binaries to succeed; windows and docker may be skipped based on inputs. + # Require binaries and linux to succeed; windows and docker may be skipped based on inputs. # Each optional job must succeed if it ran — a failure means incomplete release artifacts. # see: https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution - if: ${{ !cancelled() && needs.goreleaser-binaries.result == 'success' && (needs.goreleaser-windows.result == 'success' || needs.goreleaser-windows.result == 'skipped') && (needs.goreleaser-docker.result == 'success' || needs.goreleaser-docker.result == 'skipped') }} - needs: [determine-workflows-ref, goreleaser-binaries, goreleaser-windows, goreleaser-docker] + if: ${{ !cancelled() && needs.goreleaser-binaries.result == 'success' && needs.goreleaser-linux.result == 'success' && (needs.goreleaser-windows.result == 'success' || needs.goreleaser-windows.result == 'skipped') && (needs.goreleaser-docker.result == 'success' || needs.goreleaser-docker.result == 'skipped') }} + needs: [determine-workflows-ref, goreleaser-binaries, goreleaser-linux, goreleaser-windows, goreleaser-docker] outputs: merged_manifest: ${{ steps.export-manifest.outputs.merged_manifest }} manifest_url: ${{ steps.upload-manifest.outputs.manifest_url }} @@ -1256,10 +1535,11 @@ jobs: go-version-file: "_workflows/go.mod" cache: false - - name: Merge binaries, Windows, and images manifests + - name: Merge binaries, linux, Windows, and images manifests working-directory: _workflows env: BINARIES_MANIFEST: ${{ needs.goreleaser-binaries.outputs.binaries_manifest }} + LINUX_MANIFEST: ${{ needs.goreleaser-linux.outputs.linux_manifest }} WINDOWS_MANIFEST: ${{ needs.goreleaser-windows.outputs.windows_manifest }} IMAGES_MANIFEST: ${{ needs.goreleaser-docker.outputs.images_manifest }} OUTPUT_DIR: _output @@ -1267,6 +1547,7 @@ jobs: mkdir -p "${OUTPUT_DIR}" go run ./cmd/merge-manifests \ -binaries-manifest "$BINARIES_MANIFEST" \ + -linux-manifest "$LINUX_MANIFEST" \ -windows-manifest "$WINDOWS_MANIFEST" \ -images-manifest "$IMAGES_MANIFEST" \ | tee "${OUTPUT_DIR}/manifest.json" @@ -1284,6 +1565,7 @@ jobs: working-directory: _workflows/_output env: BINARIES_CHECKSUMS: ${{ needs.goreleaser-binaries.outputs.binaries_checksums }} + LINUX_CHECKSUMS: ${{ needs.goreleaser-linux.outputs.linux_checksums }} WINDOWS_MANIFEST: ${{ needs.goreleaser-windows.outputs.windows_manifest }} REPO_NAME: ${{ github.event.repository.name }} VERSION: ${{ inputs.tag }} @@ -1295,10 +1577,19 @@ jobs: VERSION_NO_V="${VERSION#v}" CHECKSUMS_FILE="${REPO_NAME}_${VERSION_NO_V}_checksums.txt" - # Start with binaries checksums + # Start with the darwin checksums from the binaries job echo "Creating unified checksums file: $CHECKSUMS_FILE" echo "$BINARIES_CHECKSUMS" > "./${CHECKSUMS_FILE}" + # Append the linux checksums. Each build job emits a checksums file covering only + # its own archives, so this concatenation is what makes the file complete. + if [ -z "$LINUX_CHECKSUMS" ]; then + echo "::error::Linux checksums are empty - the linux build job did not report any" + exit 1 + fi + echo "Appending linux checksums..." + echo "$LINUX_CHECKSUMS" >> "./${CHECKSUMS_FILE}" + # Append Windows asset hashes from manifest # Format: if [ -n "$WINDOWS_MANIFEST" ] && [ "$WINDOWS_MANIFEST" != "{}" ]; then @@ -1463,6 +1754,105 @@ jobs: --body "manifest.json.sigstore.json" \ --content-type "application/json" + # GoReleaser only uploaded the archives from its own run, which is now darwin-only, + # and a checksums file covering just those. Restore the release to its full contents: + # the linux archives, and the unified checksums file built above. + - name: Download staged linux archives + uses: actions/download-artifact@v4 + with: + name: linux-release-archives + path: _gh_release_assets + + - name: Attach linux archives and unified checksums to the GitHub release + working-directory: _workflows + shell: bash + env: + GH_TOKEN: ${{ secrets.RELENG_GITHUB_TOKEN }} + RELEASE_TAG: ${{ inputs.tag }} + REPO: ${{ github.repository }} + REPO_NAME: ${{ github.event.repository.name }} + VERSION: ${{ inputs.tag }} + run: | + set -euo pipefail + + shopt -s nullglob + archives=(../_gh_release_assets/*.tar.gz) + if [ ${#archives[@]} -eq 0 ]; then + echo "::error::No linux archives were staged for the GitHub release" + exit 1 + fi + + VERSION_NO_V="${VERSION#v}" + CHECKSUMS_FILE="_output/${REPO_NAME}_${VERSION_NO_V}_checksums.txt" + + # --clobber replaces the darwin-only checksums file GoReleaser already attached. + gh release upload "$RELEASE_TAG" "${archives[@]}" "$CHECKSUMS_FILE" \ + --repo "$REPO" --clobber + + echo "✅ Attached ${#archives[@]} linux archives and the unified checksums file" + + # The Homebrew formula used to be rendered by GoReleaser during the binaries run. + # That run now only sees the darwin archives, so the formula is rendered here from the + # merged manifest, where every platform's filename and hash is known. Publishing runs + # last so a tap failure cannot leave the release manifest unpublished. + - name: Generate Homebrew formula + if: inputs.brew == true + working-directory: _workflows + shell: bash + run: | + set -euo pipefail + go run ./cmd/generate-brew-formula \ + -manifest _output/manifest.json \ + -output _output/formula.rb + cat _output/formula.rb + + - name: Checkout Homebrew tap + if: inputs.brew == true + uses: actions/checkout@v5 + with: + path: _tap + repository: conductorone/${{ needs.determine-workflows-ref.outputs.brew_tap }} + token: ${{ secrets.RELENG_GITHUB_TOKEN }} + # Unlike every other checkout here, credentials are persisted: this is the one + # tree the release pushes a commit back to. + persist-credentials: true + + - name: Publish Homebrew formula + if: inputs.brew == true + working-directory: _tap + shell: bash + env: + REPO_NAME: ${{ github.event.repository.name }} + VERSION: ${{ inputs.tag }} + run: | + set -euo pipefail + mkdir -p Formula + cp ../_workflows/_output/formula.rb "Formula/${REPO_NAME}.rb" + + if [ -z "$(git status --porcelain)" ]; then + echo "Formula for ${REPO_NAME} ${VERSION} is already up to date" + exit 0 + fi + + git config user.name "conductorone-releng" + git config user.email "releng@conductorone.com" + git add "Formula/${REPO_NAME}.rb" + git commit -m "Brew formula update for ${REPO_NAME} version ${VERSION}" + + # Connectors release independently but share one tap, so a concurrent release can + # land between our fetch and push. Rebase and retry rather than failing the run. + for attempt in 1 2 3; do + if git push; then + echo "✅ Published Formula/${REPO_NAME}.rb" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); rebasing onto the latest tap" + git pull --rebase + done + + echo "::error::Failed to push the Homebrew formula after 3 attempts" + exit 1 + # ================================================================ # Registry API: record release after release manifest publication. # This is the sole release metadata recording path. @@ -1658,6 +2048,7 @@ jobs: [ determine-workflows-ref, goreleaser-binaries, + goreleaser-linux, goreleaser-windows, goreleaser-docker, publish-release-manifest, diff --git a/cmd/generate-brew-formula/main.go b/cmd/generate-brew-formula/main.go new file mode 100644 index 0000000..827958c --- /dev/null +++ b/cmd/generate-brew-formula/main.go @@ -0,0 +1,208 @@ +// Command generate-brew-formula renders a Homebrew formula from a merged release manifest. +// +// GoReleaser used to render this formula as part of the binaries run, but that run now only +// builds the darwin targets (linux moved to its own job so it can compile in parallel), so +// GoReleaser can no longer see the linux archives the tap's on_linux blocks need. This tool +// renders the formula from the merged manifest instead, where every platform is present. +// +// Download URLs point at the CDN rather than GitHub release assets: the CDN is the canonical +// distribution channel and is what the connector registry records. +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "strings" + "text/template" + + "google.golang.org/protobuf/encoding/protojson" + + pb "github.com/ConductorOne/github-workflows/pb/artifacts/v1" +) + +// platformBlock is one `url`/`sha256` pair rendered inside an on_macos/on_linux block. +type platformBlock struct { + Condition string + URL string + SHA256 string +} + +type formulaData struct { + Class string + Binary string + Homepage string + Version string + MacOS []platformBlock + Linux []platformBlock +} + +// formulaTemplate mirrors the layout GoReleaser produced so the tap diff stays reviewable. +const formulaTemplate = `# typed: false +# frozen_string_literal: true + +# This file was generated by ConductorOne/github-workflows. DO NOT EDIT. +class {{ .Class }} < Formula + desc "" + homepage "{{ .Homepage }}" + version "{{ .Version }}" +{{ if .MacOS }} + on_macos do +{{- range .MacOS }} + if {{ .Condition }} + url "{{ .URL }}" + sha256 "{{ .SHA256 }}" + + def install + bin.install "{{ $.Binary }}" + end + end +{{- end }} + end +{{- end }} +{{ if .Linux }} + on_linux do +{{- range .Linux }} + if {{ .Condition }} + url "{{ .URL }}" + sha256 "{{ .SHA256 }}" + + def install + bin.install "{{ $.Binary }}" + end + end +{{- end }} + end +{{- end }} + + test do + system "#{bin}/{{ .Binary }} -v" + end +end +` + +func main() { + var ( + manifestPath string + outputPath string + homepage string + ) + flag.StringVar(&manifestPath, "manifest", "", "Path to the merged manifest.json") + flag.StringVar(&outputPath, "output", "", "Path to write the rendered formula") + flag.StringVar(&homepage, "homepage", "https://conductorone.com", "Formula homepage") + flag.Parse() + + if manifestPath == "" || outputPath == "" { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: manifest and output are required\n") + os.Exit(1) + } + + raw, err := os.ReadFile(manifestPath) + if err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: reading manifest: %v\n", err) + os.Exit(1) + } + + manifest := &pb.Manifest{} + if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(raw, manifest); err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: parsing manifest: %v\n", err) + os.Exit(1) + } + + name := manifest.GetName() + if name == "" { + fmt.Fprintf(os.Stderr, "generate-brew-formula: ::error::Manifest is missing 'name'\n") + os.Exit(1) + } + + data := formulaData{ + Class: formulaClass(name), + Binary: name, + Homepage: homepage, + Version: strings.TrimPrefix(manifest.GetSemver(), "v"), + } + + // Ordering matches GoReleaser's output: intel before arm within each OS block. + macOSPlatforms := []struct{ key, condition string }{ + {"darwin-amd64", "Hardware::CPU.intel?"}, + {"darwin-arm64", "Hardware::CPU.arm?"}, + } + linuxPlatforms := []struct{ key, condition string }{ + {"linux-amd64", "Hardware::CPU.intel? && Hardware::CPU.is_64_bit?"}, + {"linux-arm64", "Hardware::CPU.arm? && Hardware::CPU.is_64_bit?"}, + } + + assets := manifest.GetAssets() + for _, p := range macOSPlatforms { + if block, ok := blockFor(assets, p.key, p.condition); ok { + data.MacOS = append(data.MacOS, block) + } + } + for _, p := range linuxPlatforms { + if block, ok := blockFor(assets, p.key, p.condition); ok { + data.Linux = append(data.Linux, block) + } + } + + // A formula with no platforms would silently uninstall the connector for every user. + if len(data.MacOS) == 0 && len(data.Linux) == 0 { + fmt.Fprintf(os.Stderr, "generate-brew-formula: ::error::Manifest contains no darwin or linux assets\n") + os.Exit(1) + } + + rendered, err := renderFormula(data) + if err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: rendering formula: %v\n", err) + os.Exit(1) + } + + if err := os.WriteFile(outputPath, rendered, 0o644); err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: writing formula: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "✅ Generated formula for %s (%d macOS, %d linux platforms)\n", + name, len(data.MacOS), len(data.Linux)) +} + +// renderFormula renders the Homebrew formula for data. +func renderFormula(data formulaData) ([]byte, error) { + tmpl, err := template.New("formula").Parse(formulaTemplate) + if err != nil { + return nil, fmt.Errorf("parsing template: %w", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return nil, fmt.Errorf("executing template: %w", err) + } + return buf.Bytes(), nil +} + +// blockFor builds the template block for one platform, reporting false when the manifest +// has no asset for it (for example a connector that does not ship a darwin build). +func blockFor(assets map[string]*pb.Asset, key, condition string) (platformBlock, bool) { + asset, ok := assets[key] + if !ok || asset.GetHref() == "" || asset.GetSha256() == "" { + return platformBlock{}, false + } + return platformBlock{ + Condition: condition, + URL: asset.GetHref(), + SHA256: asset.GetSha256(), + }, true +} + +// formulaClass converts a repository name into the Ruby class name Homebrew expects, +// matching GoReleaser's behaviour: "baton-okta" becomes "BatonOkta". +func formulaClass(name string) string { + var b strings.Builder + for _, part := range strings.FieldsFunc(name, func(r rune) bool { + return r == '-' || r == '_' || r == '.' || r == ' ' + }) { + b.WriteString(strings.ToUpper(part[:1])) + b.WriteString(part[1:]) + } + return b.String() +} diff --git a/cmd/generate-brew-formula/main_test.go b/cmd/generate-brew-formula/main_test.go new file mode 100644 index 0000000..6d37e51 --- /dev/null +++ b/cmd/generate-brew-formula/main_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "strings" + "testing" + + pb "github.com/ConductorOne/github-workflows/pb/artifacts/v1" +) + +func asset(href, sha string) *pb.Asset { + return pb.Asset_builder{Href: &href, Sha256: &sha}.Build() +} + +func TestFormulaClass(t *testing.T) { + for name, want := range map[string]string{ + "baton-okta": "BatonOkta", + "baton-aws": "BatonAws", + "bridge-client": "BridgeClient", + "cone": "Cone", + "baton_some_thing": "BatonSomeThing", + "baton-multi-part-x": "BatonMultiPartX", + } { + if got := formulaClass(name); got != want { + t.Errorf("formulaClass(%q) = %q, want %q", name, got, want) + } + } +} + +func TestBlockForSkipsIncompleteAssets(t *testing.T) { + assets := map[string]*pb.Asset{ + "linux-amd64": asset("https://example.test/a.tar.gz", "abc"), + "linux-arm64": asset("", "abc"), + "darwin-arm64": asset("https://example.test/b.zip", ""), + } + + if _, ok := blockFor(assets, "linux-amd64", "cond"); !ok { + t.Error("complete asset should produce a block") + } + // An asset missing an href or hash would render a formula that cannot install. + if _, ok := blockFor(assets, "linux-arm64", "cond"); ok { + t.Error("asset without href should be skipped") + } + if _, ok := blockFor(assets, "darwin-arm64", "cond"); ok { + t.Error("asset without sha256 should be skipped") + } + if _, ok := blockFor(assets, "windows-amd64", "cond"); ok { + t.Error("absent asset should be skipped") + } +} + +func TestFormulaClassEmptySegments(t *testing.T) { + // FieldsFunc drops empty segments, so a stray separator must not panic on part[:1]. + if got := formulaClass("baton--okta-"); got != "BatonOkta" { + t.Errorf("formulaClass with repeated separators = %q", got) + } +} + +func TestRenderFormulaMatchesTapLayout(t *testing.T) { + out := mustRender(t, formulaData{ + Class: "BatonOkta", + Binary: "baton-okta", + Homepage: "https://conductorone.com", + Version: "0.5.36", + MacOS: []platformBlock{ + {Condition: "Hardware::CPU.intel?", URL: "https://cdn.test/darwin-amd64.zip", SHA256: "aaa"}, + }, + Linux: []platformBlock{ + {Condition: "Hardware::CPU.intel? && Hardware::CPU.is_64_bit?", URL: "https://cdn.test/linux-amd64.tar.gz", SHA256: "bbb"}, + }, + }) + + for _, needle := range []string{ + "class BatonOkta < Formula", + `version "0.5.36"`, + "on_macos do", + "on_linux do", + `url "https://cdn.test/darwin-amd64.zip"`, + `sha256 "bbb"`, + `bin.install "baton-okta"`, + `system "#{bin}/baton-okta -v"`, + } { + if !strings.Contains(out, needle) { + t.Errorf("rendered formula missing %q\n%s", needle, out) + } + } +} + +func TestRenderFormulaOmitsEmptyOSBlock(t *testing.T) { + // A connector with no darwin build must not emit a dangling empty on_macos block. + out := mustRender(t, formulaData{ + Class: "BatonLinuxOnly", + Binary: "baton-linux-only", + Version: "1.0.0", + Linux: []platformBlock{ + {Condition: "Hardware::CPU.intel? && Hardware::CPU.is_64_bit?", URL: "https://cdn.test/l.tar.gz", SHA256: "ccc"}, + }, + }) + + if strings.Contains(out, "on_macos") { + t.Errorf("formula with no darwin assets should omit on_macos\n%s", out) + } + if !strings.Contains(out, "on_linux do") { + t.Errorf("formula should retain on_linux\n%s", out) + } +} + +func mustRender(t *testing.T, data formulaData) string { + t.Helper() + out, err := renderFormula(data) + if err != nil { + t.Fatalf("renderFormula: %v", err) + } + return string(out) +} diff --git a/cmd/merge-manifests/main.go b/cmd/merge-manifests/main.go index 95a4e64..58ef899 100644 --- a/cmd/merge-manifests/main.go +++ b/cmd/merge-manifests/main.go @@ -21,10 +21,12 @@ const ( func main() { var ( binariesManifest string + linuxManifest string imagesManifest string windowsManifest string ) flag.StringVar(&binariesManifest, "binaries-manifest", "", "JSON string of binaries manifest") + flag.StringVar(&linuxManifest, "linux-manifest", "", "JSON string of linux manifest (optional)") flag.StringVar(&imagesManifest, "images-manifest", "", "JSON string of images manifest (optional)") flag.StringVar(&windowsManifest, "windows-manifest", "", "JSON string of Windows assets manifest (optional)") flag.Parse() @@ -51,6 +53,46 @@ func main() { os.Exit(1) } + // Merge linux assets if present. + // The linux job runs the same generate-manifest tool, so this is a full manifest rather + // than the bare asset map the Windows job emits. Only its assets are merged; the + // top-level fields (version, semver, hrefs) already come from the binaries manifest. + if linuxManifest != "" && linuxManifest != "{}" { + linuxParsed := &pb.Manifest{} + if err := opts.Unmarshal([]byte(linuxManifest), linuxParsed); err != nil { + fmt.Fprintf(os.Stderr, "merge-manifests: ::error::Invalid JSON in linux_manifest output\n") + fmt.Fprintf(os.Stderr, "merge-manifests: Raw content:\n%s\n", linuxManifest) + fmt.Fprintf(os.Stderr, "merge-manifests: Error: %v\n", err) + os.Exit(1) + } + + assets := manifest.GetAssets() + if assets == nil { + assets = make(map[string]*pb.Asset) + manifest.SetAssets(assets) + } + + added := 0 + for key, asset := range linuxParsed.GetAssets() { + // Each build job generates a partial checksums file covering only its own + // archives. publish-release-manifest concatenates them and rewrites this + // entry, so the linux job's copy would only overwrite it with a partial hash. + if key == "checksums" { + continue + } + assets[key] = asset + added++ + } + + if added == 0 { + fmt.Fprintf(os.Stderr, "merge-manifests: ::error::linux_manifest contained no assets\n") + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "✅ Added %d linux assets to manifest\n", added) + } else { + fmt.Fprintln(os.Stderr, "ℹ️ No linux assets to add to manifest") + } + // Marshal options with frontend consumption in mind. Ensures all fields are present for predictable structure. marshalOpts := protojson.MarshalOptions{ Multiline: true, diff --git a/scripts/test-release-config-templates.py b/scripts/test-release-config-templates.py index 4d2f3d1..2a38d45 100755 --- a/scripts/test-release-config-templates.py +++ b/scripts/test-release-config-templates.py @@ -10,6 +10,7 @@ ROOT = Path(__file__).resolve().parent.parent TEMPLATES = { "binaries": ROOT / "templates/.goreleaser-binaries-template.yaml.tmpl", + "linux": ROOT / "templates/.goreleaser-linux-template.yaml.tmpl", "windows": ROOT / "templates/.goreleaser-windows-template.yaml.tmpl", "oci": ROOT / "templates/.goreleaser-docker-oci-template.yaml.tmpl", "lambda": ROOT / "templates/.goreleaser-docker-lambda-template.yaml.tmpl", @@ -56,11 +57,28 @@ def verify_case(go_main_package: str, brew_tap: str) -> None: } rendered = {name: render(path, values) for name, path in TEMPLATES.items()} - assert_main(rendered["binaries"], go_main_package, 3, "binaries template") + # The binaries template covers darwin amd64 + arm64; linux moved to its own template + # so it can build on an ubuntu runner in parallel. + assert_main(rendered["binaries"], go_main_package, 2, "binaries template") + assert_main(rendered["linux"], go_main_package, 1, "linux template") assert_main(rendered["windows"], go_main_package, 1, "windows template") assert_main(rendered["oci"], go_main_package, 1, "OCI template") assert_main(rendered["lambda"], go_main_package, 1, "Lambda template") - assert_contains(rendered["binaries"], f'name: "{brew_tap}"', "binaries template") + + # Each build job must own a disjoint set of GOOS values, otherwise two jobs would + # publish the same archive to the same immutable S3 key. + assert_contains(rendered["binaries"], "- darwin", "binaries template") + assert_contains(rendered["linux"], "- linux", "linux template") + if "- linux" in rendered["binaries"]: + raise AssertionError("binaries template still builds linux targets") + if "- darwin" in rendered["linux"]: + raise AssertionError("linux template must not build darwin targets") + + # The formula is rendered by cmd/generate-brew-formula from the merged manifest; + # GoReleaser can no longer see every platform, so no template may declare brews. + for name, text in rendered.items(): + if "brews:" in text: + raise AssertionError(f"{name} template still declares a brews block") def main() -> int: diff --git a/scripts/test-release-workflow-tag-pin.sh b/scripts/test-release-workflow-tag-pin.sh index 8b7604e..8b503a2 100755 --- a/scripts/test-release-workflow-tag-pin.sh +++ b/scripts/test-release-workflow-tag-pin.sh @@ -17,6 +17,7 @@ assert_tag_pin() { } assert_tag_pin goreleaser-binaries "Run GoReleaser" +assert_tag_pin goreleaser-linux "Run GoReleaser for linux" assert_tag_pin goreleaser-windows "Run GoReleaser for Windows" assert_tag_pin goreleaser-docker "Run GoReleaser for Docker OCI" assert_tag_pin goreleaser-docker "Run GoReleaser for Lambda" diff --git a/templates/.goreleaser-binaries-template.yaml.tmpl b/templates/.goreleaser-binaries-template.yaml.tmpl index 922c093..c6326b1 100644 --- a/templates/.goreleaser-binaries-template.yaml.tmpl +++ b/templates/.goreleaser-binaries-template.yaml.tmpl @@ -1,18 +1,10 @@ -## Binary template for signed artifacts, pushes to public registry (S3 bucket) +## macOS binary template for signed + notarized artifacts, pushes to public registry (S3 bucket) +## Note: linux builds moved to the dedicated goreleaser-linux job so they can compile in +## parallel on an ubuntu runner; the macOS runner only handles targets that need codesign/notarize. +## Note: Windows builds moved to dedicated goreleaser-windows job for MSI support. version: 2 project_name: "${REPO_NAME}" builds: - - binary: "${REPO_NAME}" - env: - - CGO_ENABLED=0 - id: linux - main: "${GO_MAIN_PACKAGE}" - goos: - - linux - goarch: - - amd64 - - arm64 - # Note: Windows builds moved to dedicated goreleaser-windows job for MSI support - binary: "${REPO_NAME}" env: - CGO_ENABLED=0 @@ -38,14 +30,6 @@ builds: post: - gon ../_workflows/_generated/.gon-arm64.json archives: - - id: linux-archive - builds: - - linux - format: tar.gz - name_template: "{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}" - files: - - none* - # Note: Windows archive moved to dedicated goreleaser-windows job - id: darwin-archive builds: - macos-amd64 @@ -56,15 +40,13 @@ archives: - none* release: ids: - - linux-archive - darwin-archive snapshot: version_template: "{{ incpatch .Version }}-dev" checksum: - # Note: checksums are NOT uploaded here - they're merged with Windows hashes - # and uploaded by the record-connector-registry job + # Note: checksums are NOT uploaded here - they're merged with the linux and Windows + # hashes and uploaded by the publish-release-manifest job. ids: - - linux-archive - darwin-archive sboms: - artifacts: archive @@ -74,7 +56,6 @@ signs: cmd: cosign artifacts: archive ids: - - linux-archive - darwin-archive certificate: "{{ .Env.artifact }}.cert" args: @@ -85,19 +66,11 @@ signs: - "{{ .Env.artifact }}" env: - COSIGN_EXPERIMENTAL=1 - # Note: checksums signing moved to record-connector-registry job - # to allow merging with Windows hashes first -brews: - - repository: - owner: conductorone - name: "${BREW_TAP}" - directory: Formula - skip_upload: "${BREW_SKIP_UPLOAD}" - homepage: https://conductorone.com - test: | - system "#{bin}/${REPO_NAME} -v" - install: |- - bin.install "${REPO_NAME}" + # Note: checksums signing moved to publish-release-manifest + # to allow merging with the linux and Windows hashes first +# Note: the Homebrew formula is generated by cmd/generate-brew-formula in the +# publish-release-manifest job. GoReleaser can only see the darwin archives from +# this run, so it can no longer render the on_linux blocks the tap requires. changelog: filters: exclude: diff --git a/templates/.goreleaser-linux-template.yaml.tmpl b/templates/.goreleaser-linux-template.yaml.tmpl new file mode 100644 index 0000000..260ca70 --- /dev/null +++ b/templates/.goreleaser-linux-template.yaml.tmpl @@ -0,0 +1,60 @@ +## Linux binary template for signed artifacts, pushes to public registry (S3 bucket). +## Split out of the binaries template so linux targets compile on an ubuntu runner in +## parallel with the macOS (darwin) job instead of contending for its 3 vCPUs. +## Invoked with --skip=publish: the darwin job owns the GitHub release. +version: 2 +project_name: "${REPO_NAME}" +builds: + - binary: "${REPO_NAME}" + env: + - CGO_ENABLED=0 + id: linux + main: "${GO_MAIN_PACKAGE}" + goos: + - linux + goarch: + - amd64 + - arm64 +archives: + - id: linux-archive + builds: + - linux + format: tar.gz + name_template: "{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}" + files: + - none* +release: + ids: + - linux-archive +snapshot: + version_template: "{{ incpatch .Version }}-dev" +checksum: + # Note: checksums are NOT uploaded here - they're merged with the darwin and Windows + # hashes and uploaded by the publish-release-manifest job. + ids: + - linux-archive +sboms: + - artifacts: archive +signs: + - id: cosign-archives + output: true + cmd: cosign + artifacts: archive + ids: + - linux-archive + certificate: "{{ .Env.artifact }}.cert" + args: + - "sign-blob" + - "--yes" + - "--output-signature={{ .Env.signature }}" + - "--output-certificate={{ .Env.certificate }}" + - "{{ .Env.artifact }}" + env: + - COSIGN_EXPERIMENTAL=1 +changelog: + filters: + exclude: + - "^docs:" + - typo + - lint + - Merge pull request