diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 73b53d9..7f65950 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -301,6 +301,8 @@ jobs: GOOS="$goos" govulncheck ./... done + # Builds the darwin targets only; linux moved to goreleaser-linux and Windows to + # goreleaser-windows. This job still owns the GitHub release and the changelog. goreleaser-binaries: needs: [determine-workflows-ref, release-preflight] runs-on: macos-latest @@ -382,8 +384,6 @@ jobs: env: REPO_NAME: ${{ github.event.repository.name }} GO_MAIN_PACKAGE: ${{ needs.determine-workflows-ref.outputs.go_main_package }} - BREW_TAP: ${{ needs.determine-workflows-ref.outputs.brew_tap }} - BREW_SKIP_UPLOAD: ${{ inputs.brew != true }} # For provenance predicate template WORKFLOWS_REF: ${{ needs.determine-workflows-ref.outputs.ref }} RELEASE_TAG: ${{ inputs.tag }} @@ -617,6 +617,285 @@ jobs: echo "${DELIM}" } >> "$GITHUB_OUTPUT" + # Linux binaries build on their own ubuntu runner so they compile in parallel with the + # macOS job rather than competing for its 3 vCPUs. The macOS runner is reserved for the + # darwin targets, which are the only ones that need codesign/notarization. + goreleaser-linux: + needs: [determine-workflows-ref, release-preflight] + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # <-- needed for cosign keyless (OIDC) + outputs: + linux_manifest: ${{ steps.generate-linux-manifest.outputs.linux_manifest }} + linux_checksums: ${{ steps.output-checksums.outputs.checksums }} + steps: + - name: Checkout caller repo + uses: actions/checkout@v5 + with: + path: _caller + repository: ${{ github.event.repository.full_name }} + ref: refs/tags/${{ inputs.tag }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify caller checkout matches release tag + working-directory: _caller + shell: bash + env: + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + tag_commit="$(git rev-list -n 1 "refs/tags/$RELEASE_TAG")" + head_commit="$(git rev-parse HEAD)" + if [ "$head_commit" != "$tag_commit" ]; then + echo "::error::Checked out $head_commit but refs/tags/$RELEASE_TAG resolves to $tag_commit" + exit 1 + fi + echo "Verified $RELEASE_TAG at $head_commit" + + - name: Checkout connector workflows + uses: actions/checkout@v5 + with: + path: _workflows + repository: ConductorOne/github-workflows + ref: ${{ needs.determine-workflows-ref.outputs.ref }} + persist-credentials: false + + - name: Derive AWS role names + id: role-names + working-directory: _workflows + shell: bash + env: + REPO_OWNER: ${{ github.event.repository.owner.login }} + REPO_NAME: ${{ github.event.repository.name }} + run: | + bash ./scripts/derive-iam-role-name.sh \ + --prefix GHA-Artifacts- \ + --suffix "${REPO_OWNER}-${REPO_NAME}" \ + --output-name gha_artifacts_role_name >> "$GITHUB_OUTPUT" + + - name: Set up Go for caller + uses: actions/setup-go@v6 + with: + go-version-file: "_caller/go.mod" + cache: false + + - name: Calculate S3 directory + id: s3-directory + shell: bash + run: | + ORG="${{ github.event.repository.owner.login }}" + REPO="${{ github.event.repository.name }}" + STORAGE_NAME="${{ inputs.release_storage_name }}" + TAG="${{ inputs.tag }}" + if [ -z "$STORAGE_NAME" ]; then + STORAGE_NAME="$REPO" + fi + echo "S3_DIRECTORY=releases/$ORG/$STORAGE_NAME/$TAG" >> "$GITHUB_OUTPUT" + + - name: Generate configs for linux + working-directory: _workflows + env: + REPO_NAME: ${{ github.event.repository.name }} + GO_MAIN_PACKAGE: ${{ needs.determine-workflows-ref.outputs.go_main_package }} + # For provenance predicate template + WORKFLOWS_REF: ${{ needs.determine-workflows-ref.outputs.ref }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + mkdir -p "${GENERATED_DIR}" + export BUILD_STARTED_ON=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + envsubst < templates/.goreleaser-linux-template.yaml.tmpl | tee "${GENERATED_DIR}/.goreleaser.linux.yaml" + envsubst < templates/.slsa-provenance-predicate-template.json.tmpl | tee "${GENERATED_DIR}/predicate.json" + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Download syft + uses: anchore/sbom-action/download-syft@v0 + + - name: Configure AWS credentials via OIDC + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: arn:aws:iam::025044153841:role/${{ steps.role-names.outputs.gha_artifacts_role_name }} + aws-region: us-west-2 + + # --skip=publish: the darwin job owns the GitHub release and the changelog. + - name: Run GoReleaser for linux + uses: goreleaser/goreleaser-action@v6 + with: + workdir: _caller + version: "~> v2.13" + args: release --clean --skip=publish --config ../_workflows/_generated/.goreleaser.linux.yaml + env: + GITHUB_TOKEN: ${{ secrets.RELENG_GITHUB_TOKEN }} + GORELEASER_CURRENT_TAG: ${{ inputs.tag }} + + - name: Verify binary module version + if: inputs.verify_module_version == true + working-directory: _caller + shell: bash + env: + RELEASE_TAG: ${{ inputs.tag }} + REPOSITORY_NAME: ${{ github.event.repository.name }} + run: | + set -euo pipefail + binary="$(find dist -type f -name "$REPOSITORY_NAME" -print -quit)" + if [ -z "$binary" ]; then + echo "::error::No generated $REPOSITORY_NAME binary found for module version verification" + exit 1 + fi + module_version="$(go version -m "$binary" | awk '$1 == "mod" { print $3; exit }')" + if [ "$module_version" != "$RELEASE_TAG" ]; then + echo "::error::Generated binary module version is '$module_version', expected '$RELEASE_TAG'" + exit 1 + fi + + - name: Generate SLSA provenance for archives + working-directory: _workflows + env: + CALLER_DIST: ../_caller/dist + shell: bash + run: | + set -euo pipefail + + PROVENANCE_COUNT=0 + + for artifact in "${CALLER_DIST}"/*.tar.gz; do + [ -f "$artifact" ] || continue + [[ "$artifact" == *checksums* ]] && continue + + BASENAME=$(basename "$artifact") + echo "Generating provenance for: $BASENAME" + cosign attest-blob \ + --yes \ + --predicate "${GENERATED_DIR}/predicate.json" \ + --type slsaprovenance1 \ + --bundle "${CALLER_DIST}/${BASENAME}.provenance.sigstore.json" \ + "$artifact" > /dev/null + echo "✅ Created ${BASENAME}.provenance.sigstore.json" + ((PROVENANCE_COUNT++)) || true + done + + echo "Generated provenance bundles: ${PROVENANCE_COUNT}" + if [ "$PROVENANCE_COUNT" -eq 0 ]; then + echo "::error::No provenance bundles were generated - this indicates a build problem" + exit 1 + fi + ls "${CALLER_DIST}"/*.provenance.sigstore.json + + - name: Sign SBOMs as attestation bundles + working-directory: _workflows + env: + CALLER_DIST: ../_caller/dist + shell: bash + run: | + set -euo pipefail + + SIGNED_COUNT=0 + + # Require every linux release archive to have an SPDX SBOM. + for artifact in "${CALLER_DIST}"/*.tar.gz; do + [ -f "$artifact" ] || continue + [[ "$artifact" == *checksums* ]] && continue + + SBOM="${artifact}.sbom.json" + if [ ! -f "$SBOM" ]; then + echo "::error::Missing SBOM for artifact: $(basename "$artifact") (expected: $SBOM)" + exit 1 + fi + + echo "Signing SBOM for: $(basename "$artifact")" + cosign attest-blob \ + --yes \ + --predicate "$SBOM" \ + --type https://spdx.dev/Document \ + --bundle "${artifact}.sbom.sigstore.json" \ + "$artifact" > /dev/null + echo "✅ Created $(basename "$artifact").sbom.sigstore.json" + ((SIGNED_COUNT++)) || true + done + + echo "Generated SBOM bundles: ${SIGNED_COUNT}" + if [ "$SIGNED_COUNT" -eq 0 ]; then + echo "::error::No linux SBOM bundles were generated - this indicates a build problem" + exit 1 + fi + ls "${CALLER_DIST}"/*.sbom.sigstore.json + + - name: Upload linux release artifacts to S3 + working-directory: _workflows + env: + S3_BUCKET: ${{ env.S3_BUCKET }} + S3_DIRECTORY: ${{ steps.s3-directory.outputs.S3_DIRECTORY }} + shell: bash + run: | + set -euo pipefail + ./scripts/upload-release-artifacts.sh \ + --bucket "$S3_BUCKET" \ + --directory "$S3_DIRECTORY" \ + --base-dir "../_caller/dist" + + - name: Set up Go for workflows + uses: actions/setup-go@v6 + with: + go-version-file: "_workflows/go.mod" + cache: false + + # Only the assets map of this manifest is merged; the top-level fields (semver, + # released_at, hrefs) come from the binaries manifest, so released-at is not passed. + - name: Generate linux manifest + id: generate-linux-manifest + working-directory: _workflows + env: + CALLER_DIST: ../_caller/dist + run: | + MANIFEST_JSON=$(go run ./cmd/generate-manifest \ + -asset-dir "${CALLER_DIST}" \ + -repo-name "${{ github.event.repository.name }}" \ + -org-name "${{ github.event.repository.owner.login }}" \ + -tag "${{ inputs.tag }}" \ + -base-url "${{ env.CDN_BASE_URL }}/${{ steps.s3-directory.outputs.S3_DIRECTORY }}") + + echo "$MANIFEST_JSON" + + { + echo "linux_manifest<> "$GITHUB_OUTPUT" + + # The darwin job creates the GitHub release, so this job cannot attach to it yet. + # Hand the archives to publish-release-manifest, which runs after both. + - name: Stage linux archives for the GitHub release + uses: actions/upload-artifact@v4 + with: + name: linux-release-archives + path: _caller/dist/*.tar.gz + if-no-files-found: error + retention-days: 1 + + - name: Output checksums for merging + id: output-checksums + working-directory: _caller + run: | + CHECKSUMS_FILE=$(ls dist/*checksums*.txt 2>/dev/null | head -1) + if [ -z "$CHECKSUMS_FILE" ]; then + echo "::error::No checksums file found" + exit 1 + fi + + echo "Found checksums file: $CHECKSUMS_FILE" + + # Use randomized delimiter to prevent injection via filenames containing "EOF" + DELIM="CHECKSUMS_$(openssl rand -hex 8)" + { + echo "checksums<<${DELIM}" + cat "$CHECKSUMS_FILE" + echo "${DELIM}" + } >> "$GITHUB_OUTPUT" + goreleaser-windows: if: inputs.msi == true needs: [determine-workflows-ref, release-preflight] @@ -1216,11 +1495,11 @@ jobs: publish-release-manifest: # Release manifest publication: manifest + checksums + S3 upload. - # Require binaries to succeed; windows and docker may be skipped based on inputs. + # Require binaries and linux to succeed; windows and docker may be skipped based on inputs. # Each optional job must succeed if it ran — a failure means incomplete release artifacts. # see: https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution - if: ${{ !cancelled() && needs.goreleaser-binaries.result == 'success' && (needs.goreleaser-windows.result == 'success' || needs.goreleaser-windows.result == 'skipped') && (needs.goreleaser-docker.result == 'success' || needs.goreleaser-docker.result == 'skipped') }} - needs: [determine-workflows-ref, goreleaser-binaries, goreleaser-windows, goreleaser-docker] + if: ${{ !cancelled() && needs.goreleaser-binaries.result == 'success' && needs.goreleaser-linux.result == 'success' && (needs.goreleaser-windows.result == 'success' || needs.goreleaser-windows.result == 'skipped') && (needs.goreleaser-docker.result == 'success' || needs.goreleaser-docker.result == 'skipped') }} + needs: [determine-workflows-ref, goreleaser-binaries, goreleaser-linux, goreleaser-windows, goreleaser-docker] outputs: merged_manifest: ${{ steps.export-manifest.outputs.merged_manifest }} manifest_url: ${{ steps.upload-manifest.outputs.manifest_url }} @@ -1256,10 +1535,11 @@ jobs: go-version-file: "_workflows/go.mod" cache: false - - name: Merge binaries, Windows, and images manifests + - name: Merge binaries, linux, Windows, and images manifests working-directory: _workflows env: BINARIES_MANIFEST: ${{ needs.goreleaser-binaries.outputs.binaries_manifest }} + LINUX_MANIFEST: ${{ needs.goreleaser-linux.outputs.linux_manifest }} WINDOWS_MANIFEST: ${{ needs.goreleaser-windows.outputs.windows_manifest }} IMAGES_MANIFEST: ${{ needs.goreleaser-docker.outputs.images_manifest }} OUTPUT_DIR: _output @@ -1267,6 +1547,7 @@ jobs: mkdir -p "${OUTPUT_DIR}" go run ./cmd/merge-manifests \ -binaries-manifest "$BINARIES_MANIFEST" \ + -linux-manifest "$LINUX_MANIFEST" \ -windows-manifest "$WINDOWS_MANIFEST" \ -images-manifest "$IMAGES_MANIFEST" \ | tee "${OUTPUT_DIR}/manifest.json" @@ -1284,6 +1565,7 @@ jobs: working-directory: _workflows/_output env: BINARIES_CHECKSUMS: ${{ needs.goreleaser-binaries.outputs.binaries_checksums }} + LINUX_CHECKSUMS: ${{ needs.goreleaser-linux.outputs.linux_checksums }} WINDOWS_MANIFEST: ${{ needs.goreleaser-windows.outputs.windows_manifest }} REPO_NAME: ${{ github.event.repository.name }} VERSION: ${{ inputs.tag }} @@ -1295,10 +1577,19 @@ jobs: VERSION_NO_V="${VERSION#v}" CHECKSUMS_FILE="${REPO_NAME}_${VERSION_NO_V}_checksums.txt" - # Start with binaries checksums + # Start with the darwin checksums from the binaries job echo "Creating unified checksums file: $CHECKSUMS_FILE" echo "$BINARIES_CHECKSUMS" > "./${CHECKSUMS_FILE}" + # Append the linux checksums. Each build job emits a checksums file covering only + # its own archives, so this concatenation is what makes the file complete. + if [ -z "$LINUX_CHECKSUMS" ]; then + echo "::error::Linux checksums are empty - the linux build job did not report any" + exit 1 + fi + echo "Appending linux checksums..." + echo "$LINUX_CHECKSUMS" >> "./${CHECKSUMS_FILE}" + # Append Windows asset hashes from manifest # Format: if [ -n "$WINDOWS_MANIFEST" ] && [ "$WINDOWS_MANIFEST" != "{}" ]; then @@ -1463,6 +1754,105 @@ jobs: --body "manifest.json.sigstore.json" \ --content-type "application/json" + # GoReleaser only uploaded the archives from its own run, which is now darwin-only, + # and a checksums file covering just those. Restore the release to its full contents: + # the linux archives, and the unified checksums file built above. + - name: Download staged linux archives + uses: actions/download-artifact@v4 + with: + name: linux-release-archives + path: _gh_release_assets + + - name: Attach linux archives and unified checksums to the GitHub release + working-directory: _workflows + shell: bash + env: + GH_TOKEN: ${{ secrets.RELENG_GITHUB_TOKEN }} + RELEASE_TAG: ${{ inputs.tag }} + REPO: ${{ github.repository }} + REPO_NAME: ${{ github.event.repository.name }} + VERSION: ${{ inputs.tag }} + run: | + set -euo pipefail + + shopt -s nullglob + archives=(../_gh_release_assets/*.tar.gz) + if [ ${#archives[@]} -eq 0 ]; then + echo "::error::No linux archives were staged for the GitHub release" + exit 1 + fi + + VERSION_NO_V="${VERSION#v}" + CHECKSUMS_FILE="_output/${REPO_NAME}_${VERSION_NO_V}_checksums.txt" + + # --clobber replaces the darwin-only checksums file GoReleaser already attached. + gh release upload "$RELEASE_TAG" "${archives[@]}" "$CHECKSUMS_FILE" \ + --repo "$REPO" --clobber + + echo "✅ Attached ${#archives[@]} linux archives and the unified checksums file" + + # The Homebrew formula used to be rendered by GoReleaser during the binaries run. + # That run now only sees the darwin archives, so the formula is rendered here from the + # merged manifest, where every platform's filename and hash is known. Publishing runs + # last so a tap failure cannot leave the release manifest unpublished. + - name: Generate Homebrew formula + if: inputs.brew == true + working-directory: _workflows + shell: bash + run: | + set -euo pipefail + go run ./cmd/generate-brew-formula \ + -manifest _output/manifest.json \ + -output _output/formula.rb + cat _output/formula.rb + + - name: Checkout Homebrew tap + if: inputs.brew == true + uses: actions/checkout@v5 + with: + path: _tap + repository: conductorone/${{ needs.determine-workflows-ref.outputs.brew_tap }} + token: ${{ secrets.RELENG_GITHUB_TOKEN }} + # Unlike every other checkout here, credentials are persisted: this is the one + # tree the release pushes a commit back to. + persist-credentials: true + + - name: Publish Homebrew formula + if: inputs.brew == true + working-directory: _tap + shell: bash + env: + REPO_NAME: ${{ github.event.repository.name }} + VERSION: ${{ inputs.tag }} + run: | + set -euo pipefail + mkdir -p Formula + cp ../_workflows/_output/formula.rb "Formula/${REPO_NAME}.rb" + + if [ -z "$(git status --porcelain)" ]; then + echo "Formula for ${REPO_NAME} ${VERSION} is already up to date" + exit 0 + fi + + git config user.name "conductorone-releng" + git config user.email "releng@conductorone.com" + git add "Formula/${REPO_NAME}.rb" + git commit -m "Brew formula update for ${REPO_NAME} version ${VERSION}" + + # Connectors release independently but share one tap, so a concurrent release can + # land between our fetch and push. Rebase and retry rather than failing the run. + for attempt in 1 2 3; do + if git push; then + echo "✅ Published Formula/${REPO_NAME}.rb" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); rebasing onto the latest tap" + git pull --rebase + done + + echo "::error::Failed to push the Homebrew formula after 3 attempts" + exit 1 + # ================================================================ # Registry API: record release after release manifest publication. # This is the sole release metadata recording path. @@ -1658,6 +2048,7 @@ jobs: [ determine-workflows-ref, goreleaser-binaries, + goreleaser-linux, goreleaser-windows, goreleaser-docker, publish-release-manifest, diff --git a/cmd/generate-brew-formula/main.go b/cmd/generate-brew-formula/main.go new file mode 100644 index 0000000..827958c --- /dev/null +++ b/cmd/generate-brew-formula/main.go @@ -0,0 +1,208 @@ +// Command generate-brew-formula renders a Homebrew formula from a merged release manifest. +// +// GoReleaser used to render this formula as part of the binaries run, but that run now only +// builds the darwin targets (linux moved to its own job so it can compile in parallel), so +// GoReleaser can no longer see the linux archives the tap's on_linux blocks need. This tool +// renders the formula from the merged manifest instead, where every platform is present. +// +// Download URLs point at the CDN rather than GitHub release assets: the CDN is the canonical +// distribution channel and is what the connector registry records. +package main + +import ( + "bytes" + "flag" + "fmt" + "os" + "strings" + "text/template" + + "google.golang.org/protobuf/encoding/protojson" + + pb "github.com/ConductorOne/github-workflows/pb/artifacts/v1" +) + +// platformBlock is one `url`/`sha256` pair rendered inside an on_macos/on_linux block. +type platformBlock struct { + Condition string + URL string + SHA256 string +} + +type formulaData struct { + Class string + Binary string + Homepage string + Version string + MacOS []platformBlock + Linux []platformBlock +} + +// formulaTemplate mirrors the layout GoReleaser produced so the tap diff stays reviewable. +const formulaTemplate = `# typed: false +# frozen_string_literal: true + +# This file was generated by ConductorOne/github-workflows. DO NOT EDIT. +class {{ .Class }} < Formula + desc "" + homepage "{{ .Homepage }}" + version "{{ .Version }}" +{{ if .MacOS }} + on_macos do +{{- range .MacOS }} + if {{ .Condition }} + url "{{ .URL }}" + sha256 "{{ .SHA256 }}" + + def install + bin.install "{{ $.Binary }}" + end + end +{{- end }} + end +{{- end }} +{{ if .Linux }} + on_linux do +{{- range .Linux }} + if {{ .Condition }} + url "{{ .URL }}" + sha256 "{{ .SHA256 }}" + + def install + bin.install "{{ $.Binary }}" + end + end +{{- end }} + end +{{- end }} + + test do + system "#{bin}/{{ .Binary }} -v" + end +end +` + +func main() { + var ( + manifestPath string + outputPath string + homepage string + ) + flag.StringVar(&manifestPath, "manifest", "", "Path to the merged manifest.json") + flag.StringVar(&outputPath, "output", "", "Path to write the rendered formula") + flag.StringVar(&homepage, "homepage", "https://conductorone.com", "Formula homepage") + flag.Parse() + + if manifestPath == "" || outputPath == "" { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: manifest and output are required\n") + os.Exit(1) + } + + raw, err := os.ReadFile(manifestPath) + if err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: reading manifest: %v\n", err) + os.Exit(1) + } + + manifest := &pb.Manifest{} + if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(raw, manifest); err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: parsing manifest: %v\n", err) + os.Exit(1) + } + + name := manifest.GetName() + if name == "" { + fmt.Fprintf(os.Stderr, "generate-brew-formula: ::error::Manifest is missing 'name'\n") + os.Exit(1) + } + + data := formulaData{ + Class: formulaClass(name), + Binary: name, + Homepage: homepage, + Version: strings.TrimPrefix(manifest.GetSemver(), "v"), + } + + // Ordering matches GoReleaser's output: intel before arm within each OS block. + macOSPlatforms := []struct{ key, condition string }{ + {"darwin-amd64", "Hardware::CPU.intel?"}, + {"darwin-arm64", "Hardware::CPU.arm?"}, + } + linuxPlatforms := []struct{ key, condition string }{ + {"linux-amd64", "Hardware::CPU.intel? && Hardware::CPU.is_64_bit?"}, + {"linux-arm64", "Hardware::CPU.arm? && Hardware::CPU.is_64_bit?"}, + } + + assets := manifest.GetAssets() + for _, p := range macOSPlatforms { + if block, ok := blockFor(assets, p.key, p.condition); ok { + data.MacOS = append(data.MacOS, block) + } + } + for _, p := range linuxPlatforms { + if block, ok := blockFor(assets, p.key, p.condition); ok { + data.Linux = append(data.Linux, block) + } + } + + // A formula with no platforms would silently uninstall the connector for every user. + if len(data.MacOS) == 0 && len(data.Linux) == 0 { + fmt.Fprintf(os.Stderr, "generate-brew-formula: ::error::Manifest contains no darwin or linux assets\n") + os.Exit(1) + } + + rendered, err := renderFormula(data) + if err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: rendering formula: %v\n", err) + os.Exit(1) + } + + if err := os.WriteFile(outputPath, rendered, 0o644); err != nil { + fmt.Fprintf(os.Stderr, "generate-brew-formula: error: writing formula: %v\n", err) + os.Exit(1) + } + + fmt.Fprintf(os.Stderr, "✅ Generated formula for %s (%d macOS, %d linux platforms)\n", + name, len(data.MacOS), len(data.Linux)) +} + +// renderFormula renders the Homebrew formula for data. +func renderFormula(data formulaData) ([]byte, error) { + tmpl, err := template.New("formula").Parse(formulaTemplate) + if err != nil { + return nil, fmt.Errorf("parsing template: %w", err) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return nil, fmt.Errorf("executing template: %w", err) + } + return buf.Bytes(), nil +} + +// blockFor builds the template block for one platform, reporting false when the manifest +// has no asset for it (for example a connector that does not ship a darwin build). +func blockFor(assets map[string]*pb.Asset, key, condition string) (platformBlock, bool) { + asset, ok := assets[key] + if !ok || asset.GetHref() == "" || asset.GetSha256() == "" { + return platformBlock{}, false + } + return platformBlock{ + Condition: condition, + URL: asset.GetHref(), + SHA256: asset.GetSha256(), + }, true +} + +// formulaClass converts a repository name into the Ruby class name Homebrew expects, +// matching GoReleaser's behaviour: "baton-okta" becomes "BatonOkta". +func formulaClass(name string) string { + var b strings.Builder + for _, part := range strings.FieldsFunc(name, func(r rune) bool { + return r == '-' || r == '_' || r == '.' || r == ' ' + }) { + b.WriteString(strings.ToUpper(part[:1])) + b.WriteString(part[1:]) + } + return b.String() +} diff --git a/cmd/generate-brew-formula/main_test.go b/cmd/generate-brew-formula/main_test.go new file mode 100644 index 0000000..6d37e51 --- /dev/null +++ b/cmd/generate-brew-formula/main_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "strings" + "testing" + + pb "github.com/ConductorOne/github-workflows/pb/artifacts/v1" +) + +func asset(href, sha string) *pb.Asset { + return pb.Asset_builder{Href: &href, Sha256: &sha}.Build() +} + +func TestFormulaClass(t *testing.T) { + for name, want := range map[string]string{ + "baton-okta": "BatonOkta", + "baton-aws": "BatonAws", + "bridge-client": "BridgeClient", + "cone": "Cone", + "baton_some_thing": "BatonSomeThing", + "baton-multi-part-x": "BatonMultiPartX", + } { + if got := formulaClass(name); got != want { + t.Errorf("formulaClass(%q) = %q, want %q", name, got, want) + } + } +} + +func TestBlockForSkipsIncompleteAssets(t *testing.T) { + assets := map[string]*pb.Asset{ + "linux-amd64": asset("https://example.test/a.tar.gz", "abc"), + "linux-arm64": asset("", "abc"), + "darwin-arm64": asset("https://example.test/b.zip", ""), + } + + if _, ok := blockFor(assets, "linux-amd64", "cond"); !ok { + t.Error("complete asset should produce a block") + } + // An asset missing an href or hash would render a formula that cannot install. + if _, ok := blockFor(assets, "linux-arm64", "cond"); ok { + t.Error("asset without href should be skipped") + } + if _, ok := blockFor(assets, "darwin-arm64", "cond"); ok { + t.Error("asset without sha256 should be skipped") + } + if _, ok := blockFor(assets, "windows-amd64", "cond"); ok { + t.Error("absent asset should be skipped") + } +} + +func TestFormulaClassEmptySegments(t *testing.T) { + // FieldsFunc drops empty segments, so a stray separator must not panic on part[:1]. + if got := formulaClass("baton--okta-"); got != "BatonOkta" { + t.Errorf("formulaClass with repeated separators = %q", got) + } +} + +func TestRenderFormulaMatchesTapLayout(t *testing.T) { + out := mustRender(t, formulaData{ + Class: "BatonOkta", + Binary: "baton-okta", + Homepage: "https://conductorone.com", + Version: "0.5.36", + MacOS: []platformBlock{ + {Condition: "Hardware::CPU.intel?", URL: "https://cdn.test/darwin-amd64.zip", SHA256: "aaa"}, + }, + Linux: []platformBlock{ + {Condition: "Hardware::CPU.intel? && Hardware::CPU.is_64_bit?", URL: "https://cdn.test/linux-amd64.tar.gz", SHA256: "bbb"}, + }, + }) + + for _, needle := range []string{ + "class BatonOkta < Formula", + `version "0.5.36"`, + "on_macos do", + "on_linux do", + `url "https://cdn.test/darwin-amd64.zip"`, + `sha256 "bbb"`, + `bin.install "baton-okta"`, + `system "#{bin}/baton-okta -v"`, + } { + if !strings.Contains(out, needle) { + t.Errorf("rendered formula missing %q\n%s", needle, out) + } + } +} + +func TestRenderFormulaOmitsEmptyOSBlock(t *testing.T) { + // A connector with no darwin build must not emit a dangling empty on_macos block. + out := mustRender(t, formulaData{ + Class: "BatonLinuxOnly", + Binary: "baton-linux-only", + Version: "1.0.0", + Linux: []platformBlock{ + {Condition: "Hardware::CPU.intel? && Hardware::CPU.is_64_bit?", URL: "https://cdn.test/l.tar.gz", SHA256: "ccc"}, + }, + }) + + if strings.Contains(out, "on_macos") { + t.Errorf("formula with no darwin assets should omit on_macos\n%s", out) + } + if !strings.Contains(out, "on_linux do") { + t.Errorf("formula should retain on_linux\n%s", out) + } +} + +func mustRender(t *testing.T, data formulaData) string { + t.Helper() + out, err := renderFormula(data) + if err != nil { + t.Fatalf("renderFormula: %v", err) + } + return string(out) +} diff --git a/cmd/merge-manifests/main.go b/cmd/merge-manifests/main.go index 95a4e64..58ef899 100644 --- a/cmd/merge-manifests/main.go +++ b/cmd/merge-manifests/main.go @@ -21,10 +21,12 @@ const ( func main() { var ( binariesManifest string + linuxManifest string imagesManifest string windowsManifest string ) flag.StringVar(&binariesManifest, "binaries-manifest", "", "JSON string of binaries manifest") + flag.StringVar(&linuxManifest, "linux-manifest", "", "JSON string of linux manifest (optional)") flag.StringVar(&imagesManifest, "images-manifest", "", "JSON string of images manifest (optional)") flag.StringVar(&windowsManifest, "windows-manifest", "", "JSON string of Windows assets manifest (optional)") flag.Parse() @@ -51,6 +53,46 @@ func main() { os.Exit(1) } + // Merge linux assets if present. + // The linux job runs the same generate-manifest tool, so this is a full manifest rather + // than the bare asset map the Windows job emits. Only its assets are merged; the + // top-level fields (version, semver, hrefs) already come from the binaries manifest. + if linuxManifest != "" && linuxManifest != "{}" { + linuxParsed := &pb.Manifest{} + if err := opts.Unmarshal([]byte(linuxManifest), linuxParsed); err != nil { + fmt.Fprintf(os.Stderr, "merge-manifests: ::error::Invalid JSON in linux_manifest output\n") + fmt.Fprintf(os.Stderr, "merge-manifests: Raw content:\n%s\n", linuxManifest) + fmt.Fprintf(os.Stderr, "merge-manifests: Error: %v\n", err) + os.Exit(1) + } + + assets := manifest.GetAssets() + if assets == nil { + assets = make(map[string]*pb.Asset) + manifest.SetAssets(assets) + } + + added := 0 + for key, asset := range linuxParsed.GetAssets() { + // Each build job generates a partial checksums file covering only its own + // archives. publish-release-manifest concatenates them and rewrites this + // entry, so the linux job's copy would only overwrite it with a partial hash. + if key == "checksums" { + continue + } + assets[key] = asset + added++ + } + + if added == 0 { + fmt.Fprintf(os.Stderr, "merge-manifests: ::error::linux_manifest contained no assets\n") + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "✅ Added %d linux assets to manifest\n", added) + } else { + fmt.Fprintln(os.Stderr, "ℹ️ No linux assets to add to manifest") + } + // Marshal options with frontend consumption in mind. Ensures all fields are present for predictable structure. marshalOpts := protojson.MarshalOptions{ Multiline: true, diff --git a/scripts/test-release-config-templates.py b/scripts/test-release-config-templates.py index 4d2f3d1..2a38d45 100755 --- a/scripts/test-release-config-templates.py +++ b/scripts/test-release-config-templates.py @@ -10,6 +10,7 @@ ROOT = Path(__file__).resolve().parent.parent TEMPLATES = { "binaries": ROOT / "templates/.goreleaser-binaries-template.yaml.tmpl", + "linux": ROOT / "templates/.goreleaser-linux-template.yaml.tmpl", "windows": ROOT / "templates/.goreleaser-windows-template.yaml.tmpl", "oci": ROOT / "templates/.goreleaser-docker-oci-template.yaml.tmpl", "lambda": ROOT / "templates/.goreleaser-docker-lambda-template.yaml.tmpl", @@ -56,11 +57,28 @@ def verify_case(go_main_package: str, brew_tap: str) -> None: } rendered = {name: render(path, values) for name, path in TEMPLATES.items()} - assert_main(rendered["binaries"], go_main_package, 3, "binaries template") + # The binaries template covers darwin amd64 + arm64; linux moved to its own template + # so it can build on an ubuntu runner in parallel. + assert_main(rendered["binaries"], go_main_package, 2, "binaries template") + assert_main(rendered["linux"], go_main_package, 1, "linux template") assert_main(rendered["windows"], go_main_package, 1, "windows template") assert_main(rendered["oci"], go_main_package, 1, "OCI template") assert_main(rendered["lambda"], go_main_package, 1, "Lambda template") - assert_contains(rendered["binaries"], f'name: "{brew_tap}"', "binaries template") + + # Each build job must own a disjoint set of GOOS values, otherwise two jobs would + # publish the same archive to the same immutable S3 key. + assert_contains(rendered["binaries"], "- darwin", "binaries template") + assert_contains(rendered["linux"], "- linux", "linux template") + if "- linux" in rendered["binaries"]: + raise AssertionError("binaries template still builds linux targets") + if "- darwin" in rendered["linux"]: + raise AssertionError("linux template must not build darwin targets") + + # The formula is rendered by cmd/generate-brew-formula from the merged manifest; + # GoReleaser can no longer see every platform, so no template may declare brews. + for name, text in rendered.items(): + if "brews:" in text: + raise AssertionError(f"{name} template still declares a brews block") def main() -> int: diff --git a/scripts/test-release-workflow-tag-pin.sh b/scripts/test-release-workflow-tag-pin.sh index 8b7604e..8b503a2 100755 --- a/scripts/test-release-workflow-tag-pin.sh +++ b/scripts/test-release-workflow-tag-pin.sh @@ -17,6 +17,7 @@ assert_tag_pin() { } assert_tag_pin goreleaser-binaries "Run GoReleaser" +assert_tag_pin goreleaser-linux "Run GoReleaser for linux" assert_tag_pin goreleaser-windows "Run GoReleaser for Windows" assert_tag_pin goreleaser-docker "Run GoReleaser for Docker OCI" assert_tag_pin goreleaser-docker "Run GoReleaser for Lambda" diff --git a/templates/.goreleaser-binaries-template.yaml.tmpl b/templates/.goreleaser-binaries-template.yaml.tmpl index 922c093..c6326b1 100644 --- a/templates/.goreleaser-binaries-template.yaml.tmpl +++ b/templates/.goreleaser-binaries-template.yaml.tmpl @@ -1,18 +1,10 @@ -## Binary template for signed artifacts, pushes to public registry (S3 bucket) +## macOS binary template for signed + notarized artifacts, pushes to public registry (S3 bucket) +## Note: linux builds moved to the dedicated goreleaser-linux job so they can compile in +## parallel on an ubuntu runner; the macOS runner only handles targets that need codesign/notarize. +## Note: Windows builds moved to dedicated goreleaser-windows job for MSI support. version: 2 project_name: "${REPO_NAME}" builds: - - binary: "${REPO_NAME}" - env: - - CGO_ENABLED=0 - id: linux - main: "${GO_MAIN_PACKAGE}" - goos: - - linux - goarch: - - amd64 - - arm64 - # Note: Windows builds moved to dedicated goreleaser-windows job for MSI support - binary: "${REPO_NAME}" env: - CGO_ENABLED=0 @@ -38,14 +30,6 @@ builds: post: - gon ../_workflows/_generated/.gon-arm64.json archives: - - id: linux-archive - builds: - - linux - format: tar.gz - name_template: "{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}" - files: - - none* - # Note: Windows archive moved to dedicated goreleaser-windows job - id: darwin-archive builds: - macos-amd64 @@ -56,15 +40,13 @@ archives: - none* release: ids: - - linux-archive - darwin-archive snapshot: version_template: "{{ incpatch .Version }}-dev" checksum: - # Note: checksums are NOT uploaded here - they're merged with Windows hashes - # and uploaded by the record-connector-registry job + # Note: checksums are NOT uploaded here - they're merged with the linux and Windows + # hashes and uploaded by the publish-release-manifest job. ids: - - linux-archive - darwin-archive sboms: - artifacts: archive @@ -74,7 +56,6 @@ signs: cmd: cosign artifacts: archive ids: - - linux-archive - darwin-archive certificate: "{{ .Env.artifact }}.cert" args: @@ -85,19 +66,11 @@ signs: - "{{ .Env.artifact }}" env: - COSIGN_EXPERIMENTAL=1 - # Note: checksums signing moved to record-connector-registry job - # to allow merging with Windows hashes first -brews: - - repository: - owner: conductorone - name: "${BREW_TAP}" - directory: Formula - skip_upload: "${BREW_SKIP_UPLOAD}" - homepage: https://conductorone.com - test: | - system "#{bin}/${REPO_NAME} -v" - install: |- - bin.install "${REPO_NAME}" + # Note: checksums signing moved to publish-release-manifest + # to allow merging with the linux and Windows hashes first +# Note: the Homebrew formula is generated by cmd/generate-brew-formula in the +# publish-release-manifest job. GoReleaser can only see the darwin archives from +# this run, so it can no longer render the on_linux blocks the tap requires. changelog: filters: exclude: diff --git a/templates/.goreleaser-linux-template.yaml.tmpl b/templates/.goreleaser-linux-template.yaml.tmpl new file mode 100644 index 0000000..260ca70 --- /dev/null +++ b/templates/.goreleaser-linux-template.yaml.tmpl @@ -0,0 +1,60 @@ +## Linux binary template for signed artifacts, pushes to public registry (S3 bucket). +## Split out of the binaries template so linux targets compile on an ubuntu runner in +## parallel with the macOS (darwin) job instead of contending for its 3 vCPUs. +## Invoked with --skip=publish: the darwin job owns the GitHub release. +version: 2 +project_name: "${REPO_NAME}" +builds: + - binary: "${REPO_NAME}" + env: + - CGO_ENABLED=0 + id: linux + main: "${GO_MAIN_PACKAGE}" + goos: + - linux + goarch: + - amd64 + - arm64 +archives: + - id: linux-archive + builds: + - linux + format: tar.gz + name_template: "{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}" + files: + - none* +release: + ids: + - linux-archive +snapshot: + version_template: "{{ incpatch .Version }}-dev" +checksum: + # Note: checksums are NOT uploaded here - they're merged with the darwin and Windows + # hashes and uploaded by the publish-release-manifest job. + ids: + - linux-archive +sboms: + - artifacts: archive +signs: + - id: cosign-archives + output: true + cmd: cosign + artifacts: archive + ids: + - linux-archive + certificate: "{{ .Env.artifact }}.cert" + args: + - "sign-blob" + - "--yes" + - "--output-signature={{ .Env.signature }}" + - "--output-certificate={{ .Env.certificate }}" + - "{{ .Env.artifact }}" + env: + - COSIGN_EXPERIMENTAL=1 +changelog: + filters: + exclude: + - "^docs:" + - typo + - lint + - Merge pull request