From 8e9953465e894e7355efabdb4edcdde4de338c7f Mon Sep 17 00:00:00 2001 From: Gin <> Date: Sat, 17 Oct 2026 12:25:21 -0700 Subject: [PATCH] CI: cut repeated work in the C++ builds A push to main used about 240 runner-minutes, and the Ubuntu clang job took 72-89 minutes end to end. Most of that was work repeated on every run: - clang-query ran in both Ubuntu jobs on the same Linux sources, one 100-file batch at a time: 26-55 minutes per job. It now runs only in Ubuntu Default (the fastest Linux build), runs one batch per core, and lists each file once. A file compiled into several targets has one compile_commands.json entry per target, and clang-query already checks every entry of each file it's given, so passing a file twice made it parse each of those entries twice. - The macOS jobs compiled libdpp from source with our pinned brew formula: 4 minutes on Apple Silicon, 13 on Intel. The installed keg is now cached, keyed on the formula and on the versions of the libraries it links against. The step now also fails if pkg-config can't find DPP 10.1.5, because CMake would otherwise build without Discord integration and pass. - Linux and macOS builds now use ccache. Only pushes to main save the cache; PRs restore main's latest one. Windows is left out because ccache can't cache MSVC builds that write debug info to a PDB (/Zi). - install-qt-action now caches Qt instead of downloading it every job. - tabs.yml ran on pushes to every branch as well as on pull requests, so it ran twice for each PR push. It now runs on pushes to main only. Co-Authored-By: Claude Opus 5.5 --- .../workflows/cpp-ci-serial-programs-base.yml | 68 +++++++++++++++++-- .../cpp-ci-serial-programs-ubuntu-clang.yml | 1 - .../cpp-ci-serial-programs-ubuntu-default.yml | 2 + .github/workflows/tabs.yml | 6 +- 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cpp-ci-serial-programs-base.yml b/.github/workflows/cpp-ci-serial-programs-base.yml index 91a41fda9c..baead1c8d4 100644 --- a/.github/workflows/cpp-ci-serial-programs-base.yml +++ b/.github/workflows/cpp-ci-serial-programs-base.yml @@ -37,6 +37,9 @@ jobs: CMAKE_ADDITIONAL_FLAGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo" fi fi + if [[ "${{inputs.os}}" != windows* ]]; then + CMAKE_ADDITIONAL_FLAGS="$CMAKE_ADDITIONAL_FLAGS -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + fi echo "CMAKE_ADDITIONAL_FLAGS=$CMAKE_ADDITIONAL_FLAGS" >> $GITHUB_ENV if [[ "${{inputs.os}}" == windows* ]]; then @@ -71,13 +74,14 @@ jobs: aqtsource: 'git+https://github.com/miurahr/aqtinstall' version: '6.10.2' modules: 'qtmultimedia qtserialport' + cache: true - name: Install dependencies (Ubuntu) if: startsWith(inputs.os, 'ubuntu') run: | cd Arduino-Source sudo apt update - sudo apt install clang-tools libopencv-dev libsystemd-dev + sudo apt install ccache clang-tools libopencv-dev libsystemd-dev sudo apt install ./3rdPartyBinaries/libdpp-10.1.5-linux-x64.deb @@ -85,14 +89,54 @@ jobs: if: startsWith(inputs.os, 'mac') run: | cd Arduino-Source - brew install opencv@4 onnxruntime + # libsodium, openssl@3, opus and pkgconf are libdpp's dependencies, installed here so + # their versions can go into the libdpp cache key below. + brew install opencv@4 onnxruntime ccache libsodium openssl@3 opus pkgconf brew tap-new --no-git PA/libdpp brew trust PA/libdpp FORMULA_DIR="$(brew --repo PA/libdpp)/Formula" mkdir -p "$FORMULA_DIR" cp 3rdPartyBinaries/libdpp@10.1.5.rb "$FORMULA_DIR/" - brew install libdpp@10.1.5 + + echo "LIBDPP_KEG=$(brew --cellar)/libdpp@10.1.5" >> $GITHUB_ENV + echo "LIBDPP_DEPS_HASH=$(brew list --versions libsodium openssl@3 opus | shasum | cut -c1-16)" >> $GITHUB_ENV + + # Homebrew has no prebuilt bottle for our pinned libdpp formula, so `brew install` compiles + # DPP from source: about 4 minutes on Apple Silicon and 13 on Intel. Cache the installed keg + # instead. The key includes the versions of the libraries libdpp links against, so when + # Homebrew updates one of them, libdpp is rebuilt against it rather than restored. + - name: Cache libdpp (Mac) + id: libdpp-cache + if: startsWith(inputs.os, 'mac') + uses: actions/cache@v6 + with: + path: ${{env.LIBDPP_KEG}} + key: libdpp-${{inputs.os}}-${{hashFiles('Arduino-Source/3rdPartyBinaries/libdpp@10.1.5.rb')}}-${{env.LIBDPP_DEPS_HASH}} + + - name: Install libdpp (Mac) + if: startsWith(inputs.os, 'mac') + run: | + if [[ "${{steps.libdpp-cache.outputs.cache-hit}}" == "true" ]]; then + brew link libdpp@10.1.5 + else + brew install libdpp@10.1.5 + fi + # CMake quietly builds without Discord integration when it can't find DPP, so check here. + pkg-config --exact-version=10.1.5 dpp + + # Keep compiler outputs between runs, so a build only recompiles what a change affects. + # Only pushes to main save the cache: every PR restores main's latest one, and PR runs don't + # fill the repo's 10 GB cache quota with entries that no other run can read. + # Not used on Windows: ccache can't cache MSVC builds that write debug info to a PDB (/Zi). + - name: Set up ccache + if: ${{!startsWith(inputs.os, 'windows')}} + uses: hendrikmuhs/ccache-action@v1.2.24 + with: + key: ${{inputs.os}}-${{inputs.compiler}} + max-size: 2G + evict-old-files: job + save: ${{github.ref == 'refs/heads/main'}} - name: Generate binaries run: | @@ -192,11 +236,25 @@ jobs: ) EOF - files=$(jq -r '.[].file' SerialPrograms/bin/compile_commands.json) - echo "$files" | xargs --max-args=100 clang-query -p SerialPrograms/bin/ -f query.txt >> clang-query-output.txt + # Run one batch of files per core. clang-query parses every file it's given before + # matching and keeps all those ASTs in memory, so keep batches small. Each batch writes + # its own file so the output of batches running at the same time can't interleave. + # A file compiled into more than one target has one entry per target, and clang-query + # already checks every entry of each file it's given, so list each file only once. + mkdir clang-query-batches + status=0 + jq -r '.[].file' SerialPrograms/bin/compile_commands.json | sort -u | + xargs --delimiter='\n' --max-args=10 --max-procs="$(nproc)" \ + sh -c 'clang-query -p SerialPrograms/bin/ -f query.txt "$@" > "$(mktemp clang-query-batches/XXXXXX)"' sh || + status=$? + cat clang-query-batches/* > clang-query-output.txt echo "Clang query output (begin)" cat clang-query-output.txt echo "Clang query output (end)" + if [[ $status -ne 0 ]]; then + echo "::error::clang-query failed on some files (xargs exit code $status). See the output above." + exit 1 + fi if grep --silent "Match #" clang-query-output.txt; then echo "::error Forbidden code detected! It can be one of the following:" echo "::error - std::filesystem::path created from std::string. More information https://discord.com/channels/695809740428673034/1462210406616531259/1462567541825339635" diff --git a/.github/workflows/cpp-ci-serial-programs-ubuntu-clang.yml b/.github/workflows/cpp-ci-serial-programs-ubuntu-clang.yml index 3473ecd5f0..ab145e1017 100644 --- a/.github/workflows/cpp-ci-serial-programs-ubuntu-clang.yml +++ b/.github/workflows/cpp-ci-serial-programs-ubuntu-clang.yml @@ -19,5 +19,4 @@ jobs: with: os: ubuntu-26.04 compiler: clang - run-clang-query: true run-tests: true diff --git a/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml b/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml index 1d6b3efafb..e97f7f225d 100644 --- a/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml +++ b/.github/workflows/cpp-ci-serial-programs-ubuntu-default.yml @@ -19,5 +19,7 @@ jobs: with: os: ubuntu-26.04 compiler: default + # clang-query checks the source, not the compiled code, so it only needs to run in one + # job. This is the fastest Linux build (GCC with no optimization). run-clang-query: true run-tests: true diff --git a/.github/workflows/tabs.yml b/.github/workflows/tabs.yml index 4ed5eee25f..5d1c3def4e 100644 --- a/.github/workflows/tabs.yml +++ b/.github/workflows/tabs.yml @@ -1,5 +1,9 @@ name: Check for Tabs -on: [push, pull_request] +# Pushes to other branches are checked by their pull requests. +on: + push: + branches: [main] + pull_request: jobs: no-tabs: