From 3998391b75ed19bc9de269443253d347b24306cd Mon Sep 17 00:00:00 2001 From: Witness Patrol Date: Wed, 11 Nov 2026 22:55:53 -0600 Subject: [PATCH] ci(lint): shard golangci-lint across 8 LOC-balanced matrix jobs Full-repo lint took 30-45 min on a free runner and large-diff changed lint 35-50 min. Peak RSS is a ~8-9 GB floor set by the shared import closure, so sharding buys wall time rather than memory. - scripts/lint-shard.sh: deterministic LPT split of `go list ./...` by Go LOC; `check` fails if the shards do not cover every package. - scripts/lint-changed.sh: --list and --only so a shard lints only the changed packages it owns; default behaviour unchanged. - ci.yml: lint-shard matrix plus a `lint` aggregator that keeps the required check name. Lint infrastructure changes force a full lint. Each shard reports wall time and peak RSS to the job summary. Co-Authored-By: Claude Fable 5.2 --- .claude/skills/gopherstack-gates/SKILL.md | 2 +- .github/workflows/ci.yml | 69 ++++++++++++++++++++--- scripts/lint-changed.sh | 34 ++++++++++- scripts/lint-shard.sh | 58 +++++++++++++++++++ 4 files changed, 151 insertions(+), 12 deletions(-) create mode 100755 scripts/lint-shard.sh diff --git a/.claude/skills/gopherstack-gates/SKILL.md b/.claude/skills/gopherstack-gates/SKILL.md index 7514e001c6..d467c0d669 100644 --- a/.claude/skills/gopherstack-gates/SKILL.md +++ b/.claude/skills/gopherstack-gates/SKILL.md @@ -109,7 +109,7 @@ recursive locks, copied locks — relevant given the coarse-lock convention in ## What CI runs (predict failures before opening a PR) -`.github/workflows/ci.yml`: `ui-lint` → `lint` → `modernize` (`go fix -diff +`.github/workflows/ci.yml`: `ui-lint` → `lint` (8 `lint-shard` matrix jobs, LOC-balanced by `scripts/lint-shard.sh`, each restricted to changed packages unless config/lint infra changed; `lint` aggregates) → `modernize` (`go fix -diff ./...`) → `govulncheck` → `codeql` → `unit-tests` (4-way sharded: `-race -shuffle on -short -timeout 5m`, split by `awk "NR % 4 == chunk"` over `go list ./...`) → `build` (static linux binary, tags `netgo osusergo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a24239cd7..9f0108fc4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,14 @@ jobs: env: NODE_OPTIONS: --max-old-space-size=8192 - lint: + lint-shard: + name: lint-shard (${{ matrix.shard }}) runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -82,6 +88,7 @@ jobs: echo "GOTMPDIR=$base/gotmp" echo "TMPDIR=$base/gotmp" echo "GOCACHE=$base/gocache" + echo "GOLANGCI_LINT_CACHE=$base/lintcache" } >> "$GITHUB_ENV" df -h / /tmp "$base" || true @@ -100,7 +107,7 @@ jobs: done exit 1 - # Full-repo lint (go.mod changed) exceeds runner RAM; the runner is killed without swap. + # Safety net: a shard peaks near 10 GB on a 16 GB runner; kept until real-runner numbers show it is unneeded. - name: Add swap run: | sudo swapoff -a || true @@ -118,21 +125,65 @@ jobs: run: | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "v${{ steps.golangci-lint-version.outputs.version }}" - - name: Run golangci-lint (changed packages or full repo on config change) + - name: Verify shard split covers every package + if: matrix.shard == 0 + run: bash scripts/lint-shard.sh check 8 + + - name: Run golangci-lint (shard of changed packages, or of the full repo on lint/config change) + env: + GOMAXPROCS: 4 + GOMEMLIMIT: 11GiB + SHARD: ${{ matrix.shard }} run: | set -euo pipefail - if git rev-parse --verify -q origin/main >/dev/null 2>&1; then + shard_list="$RUNNER_TEMP/shard-pkgs.txt" + bash scripts/lint-shard.sh list "$SHARD" 8 > "$shard_list" + + full=1 + if [[ "$GITHUB_REF" != refs/heads/main ]] && git rev-parse --verify -q origin/main >/dev/null 2>&1; then merge_base=$(git merge-base HEAD origin/main) - if git diff --name-only "$merge_base" HEAD -- .golangci.yml go.mod | grep -q .; then - echo "lint: .golangci.yml or go.mod changed vs origin/main — running full-repo lint" - golangci-lint run --timeout 55m ./... + if git diff --name-only "$merge_base" HEAD -- .golangci.yml go.mod .github/workflows/ci.yml scripts/lint-shard.sh scripts/lint-changed.sh | grep -q .; then + echo "lint: lint config, go.mod or lint infrastructure changed vs origin/main — full lint" else - bash scripts/lint-changed.sh + full=0 fi + fi + + if [[ "$full" -eq 1 ]]; then + mapfile -t pkgs < "$shard_list" + cmd=(golangci-lint run --timeout 20m --concurrency 4 "${pkgs[@]}") else - golangci-lint run --timeout 55m ./... + bash scripts/lint-changed.sh --list --only "$shard_list" > "$RUNNER_TEMP/changed-pkgs.txt" + mapfile -t pkgs < "$RUNNER_TEMP/changed-pkgs.txt" + cmd=(env LINT_CONCURRENCY=4 LINT_TIMEOUT=20m LINT_BATCH=100 bash scripts/lint-changed.sh --only "$shard_list") fi + if [[ "${#pkgs[@]}" -eq 0 ]]; then + echo "lint: no packages to lint in shard $SHARD" + echo "lint shard $SHARD: no packages to lint" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + time_file="$RUNNER_TEMP/lint.time" + rc=0 + /usr/bin/time -v -o "$time_file" "${cmd[@]}" || rc=$? + wall=$(sed -n 's/.*Elapsed (wall clock) time.*: //p' "$time_file") + rss_kb=$(sed -n 's/.*Maximum resident set size (kbytes): //p' "$time_file") + echo "lint shard $SHARD: ${#pkgs[@]} packages, full=$full, wall $wall, peak RSS $((rss_kb / 1024)) MiB, exit $rc" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + lint: + needs: [lint-shard] + if: always() + runs-on: ubuntu-latest + steps: + - name: Require every lint shard to succeed + env: + RESULT: ${{ needs.lint-shard.result }} + run: | + echo "lint-shard result: $RESULT" + [[ "$RESULT" == success ]] + # Fails fast on tag-gated compile breaks (e2e, integration) before the slow # e2e/integration/terraform jobs run any tests -- see gopherstack-0bpp. build-check: diff --git a/scripts/lint-changed.sh b/scripts/lint-changed.sh index 41d5b7cac9..14e1805356 100755 --- a/scripts/lint-changed.sh +++ b/scripts/lint-changed.sh @@ -20,6 +20,8 @@ # scripts/lint-changed.sh --wip # only uncommitted + last commit (HEAD~1) changes (fast on huge branches) # scripts/lint-changed.sh --base # diff against instead of origin/main # scripts/lint-changed.sh FILE... # lint only these files' packages +# scripts/lint-changed.sh --list # print the changed package patterns and exit (no linting) +# scripts/lint-changed.sh --only FILE # restrict to the patterns listed in FILE (CI shards) # # Env: LINT_BATCH (15) max packages per golangci-lint/go vet invocation, # LINT_TIMEOUT (30m) per-batch limit, LINT_CONCURRENCY (from GOFLAGS -p, else nproc/2), @@ -34,6 +36,8 @@ LINT_TIMEOUT="${LINT_TIMEOUT:-30m}" LOCK_RETRIES="${LINT_LOCK_RETRIES:-20}" BASE_REF="${LINT_BASE:-}" WIP_ONLY=0 +LIST_ONLY=0 +ONLY_FILE="" args=() while [[ $# -gt 0 ]]; do @@ -46,6 +50,14 @@ while [[ $# -gt 0 ]]; do BASE_REF="$2" shift 2 ;; + --list) + LIST_ONLY=1 + shift + ;; + --only) + ONLY_FILE="$2" + shift 2 + ;; *) args+=("$1") shift @@ -53,7 +65,10 @@ while [[ $# -gt 0 ]]; do esac done -bash "$(dirname -- "${BASH_SOURCE[0]}")/gocache-trim.sh" --if-low || true +# --list: all chatter goes to stderr, only the pattern list reaches stdout (fd 3). +exec 3>&1 +[[ "$LIST_ONLY" -eq 1 ]] && exec 1>&2 +[[ "$LIST_ONLY" -eq 1 ]] || bash "$(dirname -- "${BASH_SOURCE[0]}")/gocache-trim.sh" --if-low || true lint_conc="${LINT_CONCURRENCY:-}" if [[ -z "$lint_conc" ]]; then @@ -75,7 +90,9 @@ timeout_secs() { } hard_timeout=$(( $(timeout_secs "$LINT_TIMEOUT") + 120 )) -for bin in golangci-lint go; do +bins=(golangci-lint go) +[[ "$LIST_ONLY" -eq 1 ]] && bins=(go) +for bin in "${bins[@]}"; do command -v "$bin" >/dev/null 2>&1 || { echo "lint-changed: $bin not found on PATH" >&2 exit 1 @@ -149,6 +166,19 @@ for d in "${pkgs[@]}"; do done IFS=$'\n' sorted_patterns=($(sort <<<"${patterns[*]}")); unset IFS +if [[ -n "$ONLY_FILE" ]]; then + mapfile -t sorted_patterns < <(comm -12 <(printf '%s\n' "${sorted_patterns[@]}" | LC_ALL=C sort -u) <(LC_ALL=C sort -u "$ONLY_FILE")) + if [[ ${#sorted_patterns[@]} -eq 0 ]]; then + [[ "$LIST_ONLY" -eq 1 ]] || echo "lint-changed: no changed package falls in --only set -- nothing to check." + exit 0 + fi +fi + +if [[ "$LIST_ONLY" -eq 1 ]]; then + printf '%s\n' "${sorted_patterns[@]}" >&3 + exit 0 +fi + echo "lint-changed: checking ${#sorted_patterns[@]} package(s):" printf ' %s\n' "${sorted_patterns[@]}" if [[ ${#skipped[@]} -gt 0 ]]; then diff --git a/scripts/lint-shard.sh b/scripts/lint-shard.sh new file mode 100755 index 0000000000..7bfb4f3d73 --- /dev/null +++ b/scripts/lint-shard.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Usage: lint-shard.sh list | check +# Deterministic LOC-balanced (LPT) split of the root module's packages for CI lint. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +tags=$(sed -n '/^ build-tags:/,/^[^ ]/{s/^ - //p}' .golangci.yml | paste -sd, -) + +weighted() { + local root dir n + root=$(pwd) + go list -e -tags "$tags" -f '{{.Dir}}' ./... | sort | while IFS= read -r dir; do + n=$(cat "$dir"/*.go 2>/dev/null | wc -l) + dir=${dir#"$root"} + echo "$n ./${dir#/}" + done | sed 's#^\([0-9]*\) \./$#\1 .#' +} + +assign() { + local total="$1" + weighted | LC_ALL=C sort -k1,1nr -k2,2 | awk -v t="$total" ' + BEGIN { for (i = 0; i < t; i++) load[i] = 0 } + { + best = 0 + for (i = 1; i < t; i++) if (load[i] < load[best]) best = i + load[best] += $1 + print best, $1, $2 + }' +} + +cmd="${1:-}" +case "$cmd" in + list) + [[ $# -eq 3 ]] || { echo "usage: $0 list " >&2; exit 2; } + idx="$2" total="$3" + (( idx >= 0 && idx < total )) || { echo "lint-shard: index $idx out of range 0..$((total - 1))" >&2; exit 2; } + assign "$total" | awk -v i="$idx" '$1 == i { print $3 }' | LC_ALL=C sort + ;; + check) + [[ $# -eq 2 ]] || { echo "usage: $0 check " >&2; exit 2; } + total="$2" + out=$(assign "$total") + awk '{ n[$1]++; w[$1] += $2 } END { for (i in n) printf "lint-shard: shard %d: %d packages, %d LOC\n", i, n[i], w[i] }' <<<"$out" | sort -V + expected=$(weighted | awk '{ print $2 }' | LC_ALL=C sort) + actual=$(awk '{ print $3 }' <<<"$out" | LC_ALL=C sort) + if [[ "$expected" != "$actual" ]]; then + echo "lint-shard: union of shards differs from go list:" >&2 + diff <(echo "$expected") <(echo "$actual") >&2 || true + exit 1 + fi + echo "lint-shard: OK: $(wc -l <<<"$actual") packages across $total shards, no gaps or duplicates" + ;; + *) + echo "usage: $0 list | check " >&2 + exit 2 + ;; +esac