From 1e1c938903826a903a5a2c67b65b4e6a58465853 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 16:05:02 +0800 Subject: [PATCH 01/11] ci: slim down Rocky Linux build dependencies - Remove Development Tools and unnecessary packages - Install only JDK 8, git-core and zstd with weak dependencies disabled - Use C.utf8 to avoid installing glibc-langpack-en - Resolve and validate JAVA_HOME so Gradle no longer requires which --- .github/workflows/pr-build.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index c7a5a6f4160..625ed783fab 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -113,15 +113,26 @@ jobs: env: GRADLE_USER_HOME: /github/home/.gradle - LANG: en_US.UTF-8 - LC_ALL: en_US.UTF-8 + LANG: C.utf8 + LC_ALL: C.utf8 steps: - name: Install dependencies (Rocky 8 + JDK8) run: | set -euxo pipefail - dnf -y install java-1.8.0-openjdk-devel git wget unzip which jq bc curl glibc-langpack-en - dnf -y groupinstall "Development Tools" + # Rocky 8 already provides CA certificates, JNI runtime libraries, tar and gzip. + # Its built-in C.utf8 locale provides UTF-8 without an extra language pack. + # git-core provides checkout commands; zstd supports actions/cache compression. + dnf -y --setopt=install_weak_deps=False install \ + java-1.8.0-openjdk-devel git-core zstd + # Set JAVA_HOME so the Gradle wrapper does not need which. + javac_path=$(command -v javac) + javac_real=$(readlink -f "$javac_path") + jdk_bin=$(dirname "$javac_real") + jdk_home=$(dirname "$jdk_bin") + test -x "$jdk_home/bin/java" + test -x "$jdk_home/bin/javac" + printf 'JAVA_HOME=%s\n' "$jdk_home" >> "$GITHUB_ENV" - name: Checkout code uses: actions/checkout@v5 From 5707ef1c455b1a4ce496603e3c1bcb4d73218c8b Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 16:30:28 +0800 Subject: [PATCH 02/11] ci: upload unit test logs and JUnit reports for failure diagnosis - Collect tron-test.log, rotated logs and JUnit XML across five PR build jobs - Run artifact uploads even when preceding steps fail - Use distinct artifact names per job and matrix configuration - Retain artifacts for 7 days and warn when no files are found --- .github/workflows/pr-build.yml | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 625ed783fab..8189006452c 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -63,6 +63,18 @@ jobs: java -jar "$JAR" db archive -h java -jar "$JAR" keystore --help + - name: Upload unit test logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: tron-test-logs-${{ github.job }}-jdk${{ matrix.java }}-${{ matrix.arch }} + path: | + **/logs/tron-test.log + **/logs/tron-test-*.log.zip + **/build/test-results/**/TEST-*.xml + if-no-files-found: warn + retention-days: 7 + build-ubuntu: name: Build ubuntu24 (JDK 17 / aarch64) if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'ubuntu' }} @@ -102,6 +114,18 @@ jobs: java -jar "$JAR" db archive -h java -jar "$JAR" keystore --help + - name: Upload unit test logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: tron-test-logs-${{ github.job }} + path: | + **/logs/tron-test.log + **/logs/tron-test-*.log.zip + **/build/test-results/**/TEST-*.xml + if-no-files-found: warn + retention-days: 7 + docker-build-rockylinux: name: Build rockylinux (JDK 8 / x86_64) if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'rockylinux' }} @@ -168,6 +192,18 @@ jobs: - name: Test with RocksDB engine run: ./gradlew :framework:testWithRocksDb --no-daemon + - name: Upload unit test logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: tron-test-logs-${{ github.job }} + path: | + **/logs/tron-test.log + **/logs/tron-test-*.log.zip + **/build/test-results/**/TEST-*.xml + if-no-files-found: warn + retention-days: 7 + docker-build-debian11: name: Build debian11 (JDK 8 / x86_64) if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'debian11' }} @@ -233,6 +269,18 @@ jobs: **/build/reports/jacoco/test/jacocoTestReport.xml if-no-files-found: error + - name: Upload unit test logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: tron-test-logs-${{ github.job }} + path: | + **/logs/tron-test.log + **/logs/tron-test-*.log.zip + **/build/test-results/**/TEST-*.xml + if-no-files-found: warn + retention-days: 7 + coverage-base: name: Coverage Base (JDK 8 / x86_64) if: ${{ github.event_name == 'pull_request' }} @@ -293,6 +341,18 @@ jobs: **/build/reports/jacoco/test/jacocoTestReport.xml if-no-files-found: warn + - name: Upload unit test logs + if: always() + uses: actions/upload-artifact@v6 + with: + name: tron-test-logs-${{ github.job }} + path: | + **/logs/tron-test.log + **/logs/tron-test-*.log.zip + **/build/test-results/**/TEST-*.xml + if-no-files-found: warn + retention-days: 7 + coverage-gate: name: Coverage Gate needs: [docker-build-debian11, coverage-base] From 4b6cbf63a14705f6aa76fc5995fa16e218d3f230 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 16:35:22 +0800 Subject: [PATCH 03/11] ci: remove deleted multinode workflow from PR cancellation list Remove the stale integration-test-multinode.yml reference to avoid unnecessary API requests when cancelling workflows for closed PRs. --- .github/workflows/pr-cancel.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-cancel.yml b/.github/workflows/pr-cancel.yml index 3213026d3f9..a4d46153d47 100644 --- a/.github/workflows/pr-cancel.yml +++ b/.github/workflows/pr-cancel.yml @@ -21,7 +21,6 @@ jobs: 'pr-build.yml', 'codeql.yml', 'integration-test-single-node.yml', - 'integration-test-multinode.yml', ]; const headSha = context.payload.pull_request.head.sha; const prNumber = context.payload.pull_request.number; From a22a2c7ad3ee3c458efc34b742baccae7f586e04 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 17:31:10 +0800 Subject: [PATCH 04/11] ci: capture Gradle console logs and upload HTML test reports - Save build, RocksDB test and coverage output with tee and plain console mode - Use Bash pipefail to preserve failures when capturing stdout and stderr - Include console logs and HTML test reports in diagnostic artifacts - Preserve existing test retry and base coverage failure policies --- .github/workflows/pr-build.yml | 80 +++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8189006452c..b8056094ee1 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -52,7 +52,11 @@ jobs: restore-keys: macos26-${{ matrix.arch }}-gradle- - name: Build - run: ./gradlew clean build --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test run: | @@ -63,7 +67,7 @@ jobs: java -jar "$JAR" db archive -h java -jar "$JAR" keystore --help - - name: Upload unit test logs + - name: Upload test diagnostics if: always() uses: actions/upload-artifact@v6 with: @@ -72,6 +76,8 @@ jobs: **/logs/tron-test.log **/logs/tron-test-*.log.zip **/build/test-results/**/TEST-*.xml + **/build/reports/tests/** + ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -103,7 +109,11 @@ jobs: restore-keys: ubuntu24-aarch64-gradle- - name: Build - run: ./gradlew clean build --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test run: | @@ -114,7 +124,7 @@ jobs: java -jar "$JAR" db archive -h java -jar "$JAR" keystore --help - - name: Upload unit test logs + - name: Upload test diagnostics if: always() uses: actions/upload-artifact@v6 with: @@ -123,6 +133,8 @@ jobs: **/logs/tron-test.log **/logs/tron-test-*.log.zip **/build/test-results/**/TEST-*.xml + **/build/reports/tests/** + ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -178,7 +190,11 @@ jobs: run: ./gradlew --stop || true - name: Build - run: ./gradlew clean build --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test run: | @@ -190,9 +206,13 @@ jobs: java -jar "$JAR" keystore --help - name: Test with RocksDB engine - run: ./gradlew :framework:testWithRocksDb --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - - name: Upload unit test logs + - name: Upload test diagnostics if: always() uses: actions/upload-artifact@v6 with: @@ -201,6 +221,8 @@ jobs: **/logs/tron-test.log **/logs/tron-test-*.log.zip **/build/test-results/**/TEST-*.xml + **/build/reports/tests/** + ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -244,7 +266,11 @@ jobs: debian11-x86_64-gradle- - name: Build - run: ./gradlew clean build --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test run: | @@ -256,10 +282,18 @@ jobs: java -jar "$JAR" keystore --help - name: Test with RocksDB engine - run: ./gradlew :framework:testWithRocksDb --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Generate module coverage reports - run: ./gradlew jacocoTestReport --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log - name: Upload PR coverage reports uses: actions/upload-artifact@v6 @@ -269,7 +303,7 @@ jobs: **/build/reports/jacoco/test/jacocoTestReport.xml if-no-files-found: error - - name: Upload unit test logs + - name: Upload test diagnostics if: always() uses: actions/upload-artifact@v6 with: @@ -278,6 +312,8 @@ jobs: **/logs/tron-test.log **/logs/tron-test-*.log.zip **/build/test-results/**/TEST-*.xml + **/build/reports/tests/** + ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -324,14 +360,26 @@ jobs: # this PR. The only output we need from this job is the jacoco XML for # coverage diffing, so we must not let a stale test failure block it. continue-on-error: true - run: ./gradlew clean build --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log - name: Test with RocksDB engine (base) continue-on-error: true - run: ./gradlew :framework:testWithRocksDb --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Generate module coverage reports (base) - run: ./gradlew jacocoTestReport --no-daemon + shell: bash + run: | + set -euo pipefail + mkdir -p ci-logs + ./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log - name: Upload base coverage reports uses: actions/upload-artifact@v6 @@ -341,7 +389,7 @@ jobs: **/build/reports/jacoco/test/jacocoTestReport.xml if-no-files-found: warn - - name: Upload unit test logs + - name: Upload test diagnostics if: always() uses: actions/upload-artifact@v6 with: @@ -350,6 +398,8 @@ jobs: **/logs/tron-test.log **/logs/tron-test-*.log.zip **/build/test-results/**/TEST-*.xml + **/build/reports/tests/** + ci-logs/*.log if-no-files-found: warn retention-days: 7 From 376780b00d570ac3343988f6c3a3bfadf62bf30f Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 17:34:29 +0800 Subject: [PATCH 05/11] test: remove automatic retries from framework tests Remove the test-retry plugin and retry configuration shared by test and testWithRocksDb so test failures fail the task without retrying. --- framework/build.gradle | 5 ----- 1 file changed, 5 deletions(-) diff --git a/framework/build.gradle b/framework/build.gradle index 8edd6714f95..f090b9646db 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -1,5 +1,4 @@ plugins { - id "org.gradle.test-retry" version "1.5.9" id "org.sonarqube" version "2.6" id "com.gorylenko.gradle-git-properties" version "2.4.1" } @@ -119,10 +118,6 @@ run { } def configureTestTask = { Task t -> - t.retry { - maxRetries = 5 - maxFailures = 20 - } t.testLogging { exceptionFormat = 'full' } From 679efc4f9e46fb585eb65c7f259e0fabb5f87283 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 19:44:35 +0800 Subject: [PATCH 06/11] ci: upload only essential logs on failure - Limit diagnostic artifacts to **/logs/tron-test.log and ci-logs/*.log - Upload logs only when a preceding step fails - Include base test failures tolerated by continue-on-error - Keep the existing 7-day retention period --- .github/workflows/pr-build.yml | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index b8056094ee1..ec301b26514 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -68,15 +68,12 @@ jobs: java -jar "$JAR" keystore --help - name: Upload test diagnostics - if: always() + if: failure() uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }}-jdk${{ matrix.java }}-${{ matrix.arch }} path: | **/logs/tron-test.log - **/logs/tron-test-*.log.zip - **/build/test-results/**/TEST-*.xml - **/build/reports/tests/** ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -125,15 +122,12 @@ jobs: java -jar "$JAR" keystore --help - name: Upload test diagnostics - if: always() + if: failure() uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} path: | **/logs/tron-test.log - **/logs/tron-test-*.log.zip - **/build/test-results/**/TEST-*.xml - **/build/reports/tests/** ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -213,15 +207,12 @@ jobs: ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Upload test diagnostics - if: always() + if: failure() uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} path: | **/logs/tron-test.log - **/logs/tron-test-*.log.zip - **/build/test-results/**/TEST-*.xml - **/build/reports/tests/** ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -304,15 +295,12 @@ jobs: if-no-files-found: error - name: Upload test diagnostics - if: always() + if: failure() uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} path: | **/logs/tron-test.log - **/logs/tron-test-*.log.zip - **/build/test-results/**/TEST-*.xml - **/build/reports/tests/** ci-logs/*.log if-no-files-found: warn retention-days: 7 @@ -355,6 +343,7 @@ jobs: coverage-base-x86_64-gradle- - name: Build (base) + id: base_build # Test failures on the base branch are tolerated: merge-order races can # leave the base with a pre-existing failing test that is unrelated to # this PR. The only output we need from this job is the jacoco XML for @@ -367,6 +356,7 @@ jobs: ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log - name: Test with RocksDB engine (base) + id: base_rocksdb_test continue-on-error: true shell: bash run: | @@ -390,15 +380,13 @@ jobs: if-no-files-found: warn - name: Upload test diagnostics - if: always() + # Preserve logs for test failures tolerated by continue-on-error above. + if: ${{ failure() || steps.base_build.outcome == 'failure' || steps.base_rocksdb_test.outcome == 'failure' }} uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} path: | **/logs/tron-test.log - **/logs/tron-test-*.log.zip - **/build/test-results/**/TEST-*.xml - **/build/reports/tests/** ci-logs/*.log if-no-files-found: warn retention-days: 7 From 0ecbf2d2171e1cafa235ac7bc2c61ae4b10430c8 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 9 Sep 2026 22:13:07 +0800 Subject: [PATCH 07/11] ci: increase Rocky download concurrency and retain retry config comments - Set DNF max_parallel_downloads to 10 for Rocky dependency installation - Restore the test-retry plugin and configuration as comments, keeping retries disabled --- .github/workflows/pr-build.yml | 3 ++- framework/build.gradle | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index ec301b26514..9ba1cdf55fc 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -153,7 +153,8 @@ jobs: # Rocky 8 already provides CA certificates, JNI runtime libraries, tar and gzip. # Its built-in C.utf8 locale provides UTF-8 without an extra language pack. # git-core provides checkout commands; zstd supports actions/cache compression. - dnf -y --setopt=install_weak_deps=False install \ + # Download more packages concurrently when mirror requests are slow. + dnf -y --setopt=install_weak_deps=False --setopt=max_parallel_downloads=10 install \ java-1.8.0-openjdk-devel git-core zstd # Set JAVA_HOME so the Gradle wrapper does not need which. javac_path=$(command -v javac) diff --git a/framework/build.gradle b/framework/build.gradle index f090b9646db..5e181f2f26d 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -1,4 +1,5 @@ plugins { + // id "org.gradle.test-retry" version "1.5.9" id "org.sonarqube" version "2.6" id "com.gorylenko.gradle-git-properties" version "2.4.1" } @@ -118,6 +119,10 @@ run { } def configureTestTask = { Task t -> + // t.retry { + // maxRetries = 5 + // maxFailures = 20 + // } t.testLogging { exceptionFormat = 'full' } From c1309b003f15e358c25e8595a1e2fa4af8beeaf8 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Fri, 18 Sep 2026 19:09:01 +0800 Subject: [PATCH 08/11] ci: abandon slow mirrors during Rocky dependency installation Set DNF minrate=256k and timeout=30 to abort persistently slow connections and allow fallback to another mirror. --- .github/workflows/pr-build.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 9ba1cdf55fc..e6063a3bf3a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -153,9 +153,13 @@ jobs: # Rocky 8 already provides CA certificates, JNI runtime libraries, tar and gzip. # Its built-in C.utf8 locale provides UTF-8 without an extra language pack. # git-core provides checkout commands; zstd supports actions/cache compression. - # Download more packages concurrently when mirror requests are slow. - dnf -y --setopt=install_weak_deps=False --setopt=max_parallel_downloads=10 install \ - java-1.8.0-openjdk-devel git-core zstd + # Abandon connections below 256 KiB/s for 30 seconds so DNF can try another mirror. + dnf -y \ + --setopt=install_weak_deps=False \ + --setopt=max_parallel_downloads=10 \ + --setopt=minrate=256k \ + --setopt=timeout=30 \ + install java-1.8.0-openjdk-devel git-core zstd # Set JAVA_HOME so the Gradle wrapper does not need which. javac_path=$(command -v javac) javac_real=$(readlink -f "$javac_path") From e6a0c8610591201af39cd6322806695897ea7282 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Wed, 23 Sep 2026 11:50:15 +0800 Subject: [PATCH 09/11] ci: respect Gradle build cache configuration Remove --no-build-cache from Debian and baseline coverage jobs to preserve the cache behavior introduced in #6938. --- .github/workflows/pr-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index e6063a3bf3a..8d951ae4abe 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -266,7 +266,7 @@ jobs: run: | set -euo pipefail mkdir -p ci-logs - ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log + ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test run: | @@ -282,7 +282,7 @@ jobs: run: | set -euo pipefail mkdir -p ci-logs - ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log + ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Generate module coverage reports shell: bash @@ -358,7 +358,7 @@ jobs: run: | set -euo pipefail mkdir -p ci-logs - ./gradlew clean build --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/build.log + ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Test with RocksDB engine (base) id: base_rocksdb_test @@ -367,7 +367,7 @@ jobs: run: | set -euo pipefail mkdir -p ci-logs - ./gradlew :framework:testWithRocksDb --no-daemon --no-build-cache --console=plain 2>&1 | tee ci-logs/rocksdb-test.log + ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Generate module coverage reports (base) shell: bash From 430e431c546e72d792e906d05d0d6806ba48f940 Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Sat, 10 Oct 2026 20:14:46 +0800 Subject: [PATCH 10/11] ci: improve diagnostic uploads on failure and cancellation - Upload diagnostics on failure or cancellation with a 4-minute timeout. - Add step timeouts to reserve time for diagnostic uploads. - Keep Gradle daemon shutdown failures non-fatal. - Retain job-level timeouts for base tests to avoid accepting a partial coverage baseline after a tolerated step timeout. - Preserve outcome checks for tolerated base test failures. --- .github/workflows/pr-build.yml | 38 +++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8d951ae4abe..dc64727a0c4 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -20,6 +20,8 @@ concurrency: jobs: + # Bound build/test/report steps to leave time for diagnostics before the job limit. + # Uploads after job cancellation are best-effort only. build-macos: name: Build macos26 (JDK ${{ matrix.java }} / ${{ matrix.arch }}) if: ${{ github.event_name == 'pull_request' || inputs.job == 'all' || inputs.job == 'macos' }} @@ -52,6 +54,7 @@ jobs: restore-keys: macos26-${{ matrix.arch }}-gradle- - name: Build + timeout-minutes: 45 shell: bash run: | set -euo pipefail @@ -59,6 +62,7 @@ jobs: ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test + timeout-minutes: 2 run: | set -e JAR=plugins/build/libs/Toolkit.jar @@ -68,7 +72,8 @@ jobs: java -jar "$JAR" keystore --help - name: Upload test diagnostics - if: failure() + if: ${{ failure() || cancelled() }} + timeout-minutes: 4 uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }}-jdk${{ matrix.java }}-${{ matrix.arch }} @@ -106,6 +111,7 @@ jobs: restore-keys: ubuntu24-aarch64-gradle- - name: Build + timeout-minutes: 45 shell: bash run: | set -euo pipefail @@ -113,6 +119,7 @@ jobs: ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test + timeout-minutes: 2 run: | set -e JAR=plugins/build/libs/Toolkit.jar @@ -122,7 +129,8 @@ jobs: java -jar "$JAR" keystore --help - name: Upload test diagnostics - if: failure() + if: ${{ failure() || cancelled() }} + timeout-minutes: 4 uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} @@ -186,9 +194,12 @@ jobs: rockylinux-x86_64-gradle- - name: Stop Gradle daemon + timeout-minutes: 2 + continue-on-error: true run: ./gradlew --stop || true - name: Build + timeout-minutes: 30 shell: bash run: | set -euo pipefail @@ -196,6 +207,7 @@ jobs: ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test + timeout-minutes: 2 run: | set -e JAR=plugins/build/libs/Toolkit.jar @@ -205,6 +217,7 @@ jobs: java -jar "$JAR" keystore --help - name: Test with RocksDB engine + timeout-minutes: 10 shell: bash run: | set -euo pipefail @@ -212,7 +225,8 @@ jobs: ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Upload test diagnostics - if: failure() + if: ${{ failure() || cancelled() }} + timeout-minutes: 4 uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} @@ -262,6 +276,7 @@ jobs: debian11-x86_64-gradle- - name: Build + timeout-minutes: 30 shell: bash run: | set -euo pipefail @@ -269,6 +284,7 @@ jobs: ./gradlew clean build --no-daemon --console=plain 2>&1 | tee ci-logs/build.log - name: Toolkit jar smoke test + timeout-minutes: 2 run: | set -e JAR=plugins/build/libs/Toolkit.jar @@ -278,6 +294,7 @@ jobs: java -jar "$JAR" keystore --help - name: Test with RocksDB engine + timeout-minutes: 10 shell: bash run: | set -euo pipefail @@ -285,6 +302,7 @@ jobs: ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Generate module coverage reports + timeout-minutes: 5 shell: bash run: | set -euo pipefail @@ -292,6 +310,7 @@ jobs: ./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log - name: Upload PR coverage reports + timeout-minutes: 2 uses: actions/upload-artifact@v6 with: name: jacoco-coverage-pr @@ -300,7 +319,8 @@ jobs: if-no-files-found: error - name: Upload test diagnostics - if: failure() + if: ${{ failure() || cancelled() }} + timeout-minutes: 4 uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} @@ -347,6 +367,8 @@ jobs: restore-keys: | coverage-base-x86_64-gradle- + # Use the job timeout for both base test steps. A tolerated step timeout + # could let incomplete coverage become the comparison baseline. - name: Build (base) id: base_build # Test failures on the base branch are tolerated: merge-order races can @@ -370,6 +392,7 @@ jobs: ./gradlew :framework:testWithRocksDb --no-daemon --console=plain 2>&1 | tee ci-logs/rocksdb-test.log - name: Generate module coverage reports (base) + timeout-minutes: 5 shell: bash run: | set -euo pipefail @@ -377,6 +400,7 @@ jobs: ./gradlew jacocoTestReport --no-daemon --console=plain 2>&1 | tee ci-logs/coverage.log - name: Upload base coverage reports + timeout-minutes: 2 uses: actions/upload-artifact@v6 with: name: jacoco-coverage-base @@ -386,7 +410,11 @@ jobs: - name: Upload test diagnostics # Preserve logs for test failures tolerated by continue-on-error above. - if: ${{ failure() || steps.base_build.outcome == 'failure' || steps.base_rocksdb_test.outcome == 'failure' }} + if: >- + failure() || cancelled() || + steps.base_build.outcome == 'failure' || + steps.base_rocksdb_test.outcome == 'failure' + timeout-minutes: 4 uses: actions/upload-artifact@v6 with: name: tron-test-logs-${{ github.job }} From 2f61ae1a7f28f0d935c22627be41748ded6cf6ac Mon Sep 17 00:00:00 2001 From: 3for <287494524@qq.com> Date: Mon, 12 Oct 2026 22:21:58 +0800 Subject: [PATCH 11/11] ci: add diagnostics for Gradle and JVM failures - Wrap PR build, test, coverage and Toolkit stages with a shared runner - Collect JVM crash logs, daemon logs, memory samples and OOM counters - Record test classes and methods associated with each test executor - Show failure evidence in Actions annotations and job summaries - Upload stage-specific diagnostics and available JUnit XML on failure - Preserve command exit codes when diagnostic logging fails - Use PR diagnostic scripts for base coverage builds and retain logs for tolerated failures --- .github/ci/ci-run.sh | 607 +++++++++++++++++++++++++++ .github/ci/test-progress.init.gradle | 72 ++++ .github/workflows/pr-build.yml | 119 +++--- 3 files changed, 739 insertions(+), 59 deletions(-) create mode 100755 .github/ci/ci-run.sh create mode 100644 .github/ci/test-progress.init.gradle diff --git a/.github/ci/ci-run.sh b/.github/ci/ci-run.sh new file mode 100755 index 00000000000..d8878ddfb54 --- /dev/null +++ b/.github/ci/ci-run.sh @@ -0,0 +1,607 @@ +#!/usr/bin/env bash +# Run one CI stage and keep enough evidence to explain failures that have no +# failing test case: JVM crashes, Java or kernel OOM, killed Gradle workers, +# non-test Gradle task failures, timeouts and cancellations. +# +# Usage: ci-run.sh [args...] +# +# Evidence for each stage goes to its own directory, $CI_DIAG_ROOT/ +# (default: ./ci-logs/), so a later stage never reports an earlier +# stage's crash files. Set CI_DIAG_ANNOTATION=warning for steps whose failures +# are tolerated. +# +# The command's exit status is preserved: diagnostics never turn a failure +# into a success or the other way around. The report separates what was +# observed from what it may mean. Compatible with bash 3.2 (macOS runners) and +# minimal images without findutils or procps. + +set -uo pipefail + +if [ "$#" -lt 2 ]; then + echo "usage: $0 [args...]" >&2 + exit 2 +fi + +STAGE=$1 +shift +case "$STAGE" in + *[!A-Za-z0-9_.-]* | '' | .*) + echo "invalid stage name: $STAGE" >&2 + exit 2 ;; +esac + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +ROOT=${CI_DIAG_ROOT:-$PWD/ci-logs} +DIAG_DIR=$ROOT/$STAGE +n=2 +while [ -e "$DIAG_DIR" ]; do + DIAG_DIR=$ROOT/$STAGE-$n + n=$((n + 1)) +done +mkdir -p "$DIAG_DIR" +DIAG_DIR=$(cd "$DIAG_DIR" && pwd) +export CI_DIAG_DIR=$DIAG_DIR + +CONSOLE_LOG=$DIAG_DIR/console.log +MEM_LOG=$DIAG_DIR/memory.log +JVM_LOG=$DIAG_DIR/jvms.log +JVM_LAST=$DIAG_DIR/jvms-last.txt +STATUS_FILE=$DIAG_DIR/status.txt +REPORT=$DIAG_DIR/diagnosis.md +MARKER=$DIAG_DIR/.start +SAMPLE_SECONDS=${CI_MEM_SAMPLE_SECONDS:-15} +OS=$(uname -s) + +# Write HotSpot fatal error logs to this stage's directory. JAVA_TOOL_OPTIONS +# reaches every JVM started below, including the single-use Gradle daemon, +# Gradle test workers and compiler daemons. +export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:+$JAVA_TOOL_OPTIONS }-XX:ErrorFile=$DIAG_DIR/hs_err_pid%p.log" + +log() { + echo "[ci-diag] $*" +} + +warn_diagnostics() { + local message=$1 + log "warning: $message" >&2 + if [ -n "${GITHUB_ACTIONS:-}" ]; then + printf '::warning title=CI diagnostics (%s)::%s\n' "$STAGE" "$message" + fi +} + +now() { + date -u +%H:%M:%S +} + +# ------------------------------------------------------------------ cgroups + +# Resolve this shell's memory cgroup through /proc/self/mountinfo so nested +# cgroups and non-root mounts map to the right directory. Leaves CG_DIR empty +# when the mapping is ambiguous; the report then says "unavailable" rather +# than reading an unrelated cgroup. +CG_DIR="" +CG_VERSION="" + +cgroup_path_under_mount() { + local path=$1 root=$2 mnt=$3 + if [ "$path" = "$root" ]; then + echo "$mnt" + elif [ "$root" = / ]; then + echo "$mnt$path" + else + case "$path" in + "$root"/*) echo "$mnt${path#"$root"}" ;; + esac + fi +} + +resolve_cgroup() { + [ -r /proc/self/cgroup ] && [ -r /proc/self/mountinfo ] || return 0 + local path mount dir + path=$(awk -F: '$1 == "0" && $2 == "" { print $3; exit }' /proc/self/cgroup) + if [ -n "$path" ]; then + mount=$(awk '{ for (i = 7; i <= NF; i++) if ($i == "-") { + if ($(i + 1) == "cgroup2") print $4, $5; break } }' /proc/self/mountinfo | head -1) + if [ -n "$mount" ]; then + dir=$(cgroup_path_under_mount "$path" "${mount%% *}" "${mount#* }") + if [ -n "$dir" ] && [ -f "$dir/memory.current" ]; then + CG_DIR=$dir + CG_VERSION=2 + return 0 + fi + fi + fi + # cgroup v1, or a hybrid host whose v2 hierarchy has no memory controller. + path=$(awk -F: '$2 ~ /(^|,)memory(,|$)/ { print $3; exit }' /proc/self/cgroup) + [ -n "$path" ] || return 0 + mount=$(awk '{ for (i = 7; i <= NF; i++) if ($i == "-") { + if ($(i + 1) == "cgroup" && $(i + 3) ~ /(^|,)memory(,|$)/) print $4, $5; break } }' \ + /proc/self/mountinfo | head -1) + [ -n "$mount" ] || return 0 + dir=$(cgroup_path_under_mount "$path" "${mount%% *}" "${mount#* }") + if [ -n "$dir" ] && [ -f "$dir/memory.usage_in_bytes" ]; then + CG_DIR=$dir + CG_VERSION=1 + fi +} + +read_mb() { + awk '{ if ($1 == "max" || $1 >= 9e18) print "max"; else print int($1 / 1048576) "M" }' \ + "$1" 2>/dev/null +} + +cg_usage() { + case "$CG_VERSION" in + 2) read_mb "$CG_DIR/memory.current" ;; + 1) read_mb "$CG_DIR/memory.usage_in_bytes" ;; + *) echo "-" ;; + esac +} + +cg_limit() { + case "$CG_VERSION" in + 2) read_mb "$CG_DIR/memory.max" ;; + 1) read_mb "$CG_DIR/memory.limit_in_bytes" ;; + *) echo "-" ;; + esac +} + +cg_oom_kill() { + local v="" + case "$CG_VERSION" in + 2) v=$(awk '$1 == "oom_kill" { print $2 }' "$CG_DIR/memory.events" 2>/dev/null) ;; + 1) v=$(awk '$1 == "oom_kill" { print $2 }' "$CG_DIR/memory.oom_control" 2>/dev/null) ;; + esac + echo "${v:--}" +} + +# Kernel-wide counter: on a hosted runner it covers the whole VM, so it may +# include processes outside this job. +host_oom_kill() { + local v + v=$(awk '$1 == "oom_kill" { print $2 }' /proc/vmstat 2>/dev/null) + echo "${v:--}" +} + +snapshot_memory() { + local f + { + echo "== $1 $(date -u +%Y-%m-%dT%H:%M:%SZ)" + if [ "$OS" = Linux ]; then + grep -E '^(MemTotal|MemAvailable|SwapTotal|SwapFree):' /proc/meminfo + echo "host oom_kill: $(host_oom_kill)" + if [ -n "$CG_DIR" ]; then + echo "cgroup v$CG_VERSION: $CG_DIR" + for f in memory.max memory.current memory.peak memory.events memory.swap.max \ + memory.swap.current memory.limit_in_bytes memory.usage_in_bytes \ + memory.max_usage_in_bytes memory.failcnt memory.oom_control; do + [ -r "$CG_DIR/$f" ] && echo "$f: $(tr '\n' ' ' < "$CG_DIR/$f")" + done + else + echo "cgroup: unavailable (memory cgroup of this process could not be resolved)" + fi + else + sysctl -n hw.memsize 2>/dev/null | awk '{ printf "hw.memsize: %d MB\n", $1 / 1048576 }' + sysctl vm.swapusage 2>/dev/null + memory_pressure 2>/dev/null | tail -1 + fi + } >> "$MEM_LOG" 2>&1 +} + +# ----------------------------------------------------------------- JVM list + +java_role() { + case "$1" in + *"Gradle Test Executor"*) + echo "$1" | grep -o 'Gradle Test Executor [0-9]*' | head -1 | tr ' ' '_' ;; + *GradleDaemon*) echo GradleDaemon ;; + *GradleWrapperMain*) echo GradleWrapper ;; + *GradleWorkerMain*) echo GradleWorker ;; + *KotlinCompileDaemon*) echo KotlinDaemon ;; + *) echo java ;; + esac +} + +# Prints " " for every running JVM. +java_processes() { + if [ "$OS" = Linux ]; then + local d rss cmd + for d in /proc/[0-9]*; do + [ "$(cat "$d/comm" 2>/dev/null)" = java ] || continue + rss=$(awk '/^VmRSS:/ { print $2 }' "$d/status" 2>/dev/null) + cmd=$(tr '\0' ' ' < "$d/cmdline" 2>/dev/null) + echo "${d#/proc/} ${rss:-0} $(java_role "$cmd")" + done + else + ps -axo pid=,rss=,command= 2>/dev/null | while read -r pid rss cmd; do + case "$cmd" in + */bin/java\ * | java\ *) echo "$pid $rss $(java_role "$cmd")" ;; + esac + done + fi +} + +memory_sampler() { + local prev=$DIAG_DIR/.jvms.prev cur=$DIAG_DIR/.jvms.cur i=0 ts line every + every=$((60 / SAMPLE_SECONDS)) + [ "$every" -gt 0 ] || every=1 + : > "$prev" + while :; do + ts=$(now) + java_processes | LC_ALL=C sort > "$cur" + # Full PID/role/RSS map on every sample, so a JVM that vanishes can be + # identified even if it died right after the previous sample. + awk -v ts="$ts" '{ printf "%s %s %s %dM\n", ts, $1, $3, $2 / 1024 }' "$cur" >> "$JVM_LOG" + { echo "# $ts"; cat "$cur"; } > "$JVM_LAST" + LC_ALL=C join -v1 "$prev" "$cur" 2>/dev/null | while read -r pid rss role; do + echo "$ts EXIT pid=$pid role=$role last_rss=$((rss / 1024))M" >> "$MEM_LOG" + done + line="$ts" + if [ "$OS" = Linux ]; then + line="$line avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo)M" + line="$line cgroup=$(cg_usage)/$(cg_limit) cgroup_oom_kill=$(cg_oom_kill)" + line="$line host_oom_kill=$(host_oom_kill)" + fi + line="$line jvm_total=$(awk '{ s += $2 } END { print int(s / 1024) }' "$cur")M" + line="$line top=$(sort -k2 -nr "$cur" | head -4 \ + | awk '{ printf "%s%s:%dM", sep, $3, $2 / 1024; sep = "," }')" + echo "$line" >> "$MEM_LOG" + # Stream a trend line to the job log once a minute; it survives even when + # the artifact upload never runs. + [ $((i % every)) -eq 0 ] && log "mem $line" + i=$((i + 1)) + mv "$cur" "$prev" + sleep "$SAMPLE_SECONDS" + done +} + +# --------------------------------------------------------------- diagnosis + +collect_files() { + local f gradle_home=${GRADLE_USER_HOME:-$HOME/.gradle} + # Fallback locations HotSpot uses when ErrorFile cannot be written. + for f in hs_err_pid*.log */hs_err_pid*.log /tmp/hs_err_pid*.log; do + [ -f "$f" ] && [ "$f" -nt "$MARKER" ] && cp -p "$f" "$DIAG_DIR/" 2>/dev/null + done + # Daemon logs of this stage only; tail them because they repeat build output. + for f in "$gradle_home"/daemon/*/daemon-*.out.log; do + [ -f "$f" ] && [ "$f" -nt "$MARKER" ] \ + && tail -n 20000 "$f" > "$DIAG_DIR/gradle-$(basename "$f")" + done +} + +signal_name() { + case "$1" in + 6) echo SIGABRT ;; + 9) echo SIGKILL ;; + 11) echo SIGSEGV ;; + 15) echo SIGTERM ;; + *) echo "signal $1" ;; + esac +} + +exit_text() { + if [ "$1" -gt 128 ] && [ "$1" -lt 160 ]; then + echo "$1 (128+$(($1 - 128)), $(signal_name $(($1 - 128))))" + else + echo "$1" + fi +} + +# One line per fatal-error log: owning JVM role, fatal reason, problematic frame. +hs_err_summary() { + local f role + for f in "$DIAG_DIR"/hs_err_pid*.log; do + [ -f "$f" ] || continue + role=$(java_role "$(grep -m1 '^Command Line:' "$f" 2>/dev/null)") + awk -v file="${f##*/}" -v role="$role" ' + !/^#/ { exit } + frame_next { frame = substr($0, 3); frame_next = 0; next } + /^# Problematic frame:/ { frame_next = 1; next } + /^# (Possible| )/ { next } + /^# [^ ]/ || /^# (There is insufficient memory|Native memory allocation|Out of Memory Error)/ { + if (n < 2) { sub(/^# +/, ""); reason = reason (n++ ? "; " : "") $0 } + } + END { + printf "%s [%s]: %s%s\n", file, role, (reason == "" ? "reason not found" : reason), + (frame == "" ? "" : "; frame: " frame) + }' "$f" + done +} + +diagnose() { + local rc=$1 reason=$2 observed="" hints="" f n + local cg_after host_after hs_errs workers exits roles oom_lines wrong unfinished samples + local killed=0 cg_oom=0 host_oom=0 cg_known=0 daemon_gone=0 evidence=0 + + collect_files + snapshot_memory "after ($reason)" + + obs() { observed="${observed}- $*"$'\n'; } + hint() { hints="${hints}- $*"$'\n'; } + + obs "Command exit status: $(exit_text "$rc")." + [ "$rc" -eq 137 ] && killed=1 + if [ "$reason" = interrupted ]; then + obs "The wrapper received $SIGNAL_SEEN before the command finished." + evidence=1 + fi + + if [ "$OS" = Linux ]; then + cg_after=$(cg_oom_kill) + host_after=$(host_oom_kill) + if [ -z "$CG_DIR" ]; then + obs "cgroup OOM kills: unavailable (memory cgroup not resolvable)." + elif [ "$CG_OOM_BEFORE" = - ] || [ "$cg_after" = - ]; then + obs "cgroup OOM kills: counter not exposed by cgroup v$CG_VERSION." + else + cg_known=1 + obs "cgroup OOM kills during stage: $((cg_after - CG_OOM_BEFORE)) ($CG_DIR)." + [ "$cg_after" -gt "$CG_OOM_BEFORE" ] && cg_oom=1 + fi + if [ "$HOST_OOM_BEFORE" != - ] && [ "$host_after" != - ]; then + obs "Kernel-wide OOM kills during stage: $((host_after - HOST_OOM_BEFORE)) (may include processes outside this job)." + [ "$host_after" -gt "$HOST_OOM_BEFORE" ] && host_oom=1 + fi + fi + + hs_errs=$(hs_err_summary) + if [ -n "$hs_errs" ]; then + obs "JVM fatal error log(s): $(echo "$hs_errs" | paste -sd ';' -)." + evidence=1 + fi + + workers=$(grep -oE "Process 'Gradle Test Executor [0-9]+' finished with non-zero exit value [0-9]+" \ + "$CONSOLE_LOG" 2>/dev/null | LC_ALL=C sort -u | head -5) + if [ -n "$workers" ]; then + obs "Gradle test worker(s) exited abnormally: $(echo "$workers" | sed "s/^Process '//; s/' finished with non-zero exit value / -> exit /" | paste -sd ';' -)." + echo "$workers" | grep -q 'exit value 137$' && killed=1 + evidence=1 + fi + if grep -q 'daemon disappeared unexpectedly' "$CONSOLE_LOG" 2>/dev/null; then + obs "Gradle reported that its daemon disappeared unexpectedly." + daemon_gone=1 + evidence=1 + fi + + # Test workers are recycled routinely (forkEvery), so prefer exits of the + # abnormal workers and of the Gradle daemon over the normal rotation. + roles=$( { echo "$workers" | grep -oE 'Executor [0-9]+' | sed 's/^Executor /Gradle_Test_Executor_/'; echo GradleDaemon; } \ + | sed 's/.*/role=&( |$)/' | paste -sd '|' -) + exits=$(grep ' EXIT ' "$MEM_LOG" 2>/dev/null | grep -E "$roles" | tail -5) + if [ -n "$exits" ]; then + obs "Sampler saw these JVMs disappear: $(echo "$exits" | sed 's/ EXIT / /' | paste -sd ';' -)." + else + exits=$(grep ' EXIT ' "$MEM_LOG" 2>/dev/null | tail -5) + [ -n "$exits" ] && obs "Last JVM exits seen by the sampler (routine worker recycling also appears here): $(echo "$exits" | sed 's/ EXIT / /' | paste -sd ';' -)." + fi + + oom_lines=$(grep -c 'OutOfMemoryError' "$CONSOLE_LOG" 2>/dev/null) + if [ "${oom_lines:-0}" -gt 0 ]; then + obs "Console lines mentioning OutOfMemoryError: $oom_lines, e.g. \`$(grep -m1 'OutOfMemoryError' "$CONSOLE_LOG" | cut -c1-160)\`." + evidence=1 + fi + + if [ "$IS_GRADLE" -eq 1 ]; then + if grep -qE '^BUILD (SUCCESSFUL|FAILED)' "$CONSOLE_LOG" 2>/dev/null; then + obs "Gradle completion marker: $(grep -E '^BUILD (SUCCESSFUL|FAILED)' "$CONSOLE_LOG" | tail -1)." + else + obs "Gradle completion marker (BUILD SUCCESSFUL/FAILED) not observed; read the exit status and console log together." + fi + fi + + wrong=$(awk '/^\* What went wrong:/ { on = 1; next } /^\* Try:/ { on = 0 } + on && NF { print }' "$CONSOLE_LOG" 2>/dev/null | head -15) + + unfinished=$(unfinished_suites | head -10) + [ -n "$unfinished" ] && obs "Test classes with a start but no end record: $(echo "$unfinished" | paste -sd ';' -)." + + [ "$killed" -eq 1 ] || [ "$cg_oom" -eq 1 ] || [ "$host_oom" -eq 1 ] && evidence=1 + + # Interpretation, kept separate from the facts above. Only this job's cgroup + # counter ties an OOM kill to this stage; the kernel-wide one may not. + if [ "$cg_oom" -eq 1 ] && [ "$killed" -eq 1 ]; then + hint "A process was SIGKILLed and the OOM killer fired in this job's cgroup: memory exhaustion is the likely cause. Check memory.log and jvms.log around the EXIT time." + elif [ "$cg_oom" -eq 1 ]; then + hint "The OOM killer killed a process in this job's cgroup during the stage; which process is not confirmed. A killed Gradle daemon or worker would explain a plain failure exit; check the EXIT records in memory.log and jvms.log." + elif [ "$host_oom" -eq 1 ] && [ "$cg_known" -eq 1 ]; then + hint "The kernel OOM killer fired outside this job's cgroup; it may be unrelated to this failure." + elif [ "$host_oom" -eq 1 ]; then + hint "The kernel OOM killer fired, but this job's cgroup could not be checked, so its link to this failure is unconfirmed." + fi + if [ "$killed" -eq 1 ] && [ "$cg_oom" -eq 0 ] && [ "$host_oom" -eq 0 ]; then + if [ "$cg_known" -eq 1 ]; then + hint "SIGKILL without OOM-killer evidence in this cgroup: the sender is unknown (runner, timeout handling or another process)." + else + hint "SIGKILL observed, but OOM-killer counters were not available to confirm or rule out OOM." + fi + fi + [ -n "$hs_errs" ] && hint "A JVM hit a fatal error; the reason and problematic frame above come from its hs_err header." + [ "$daemon_gone" -eq 1 ] && [ -z "$hs_errs" ] && [ "$cg_oom" -eq 0 ] \ + && hint "The Gradle daemon vanished without a fatal-error log or cgroup OOM kill: an external kill or a crash that could not write hs_err." + [ "$reason" = interrupted ] && hint "Signals usually come from a step timeout or run cancellation." + [ -n "$workers" ] && [ -z "$hs_errs" ] && [ "$killed" -eq 0 ] \ + && hint "A test worker exited with a non-zero status without a fatal-error log: possible System.exit/halt in a test, or a crash that could not write hs_err." + [ "${oom_lines:-0}" -gt 0 ] \ + && hint "OutOfMemoryError text is a clue only; confirm which JVM threw it and whether it was caught by a test." + if [ "$evidence" -eq 0 ] && [ "$IS_GRADLE" -eq 1 ]; then + hint "No crash, OOM-kill, signal or worker-exit evidence was found; the failing Gradle task in console.log is the next place to look." + elif [ "$evidence" -eq 0 ]; then + hint "No crash, OOM-kill, signal or worker-exit evidence was found; console.log shows the failing command's output." + elif [ -z "$hints" ]; then + hint "No single cause can be inferred; read the observed facts together." + fi + + { + echo "### CI diagnosis: \`$STAGE\`" + echo + echo "**Observed**" + echo + printf '%s' "$observed" + if [ -n "$wrong" ]; then + echo + echo "Gradle \"What went wrong\":" + echo '```text' + echo "$wrong" + echo '```' + fi + if [ -n "$workers" ]; then + echo + echo "Last test records of the abnormal worker(s); a worker crash can still" + echo "produce end events, so read these together with the exit status:" + echo '```text' + echo "$workers" | grep -oE 'Executor [0-9]+' | awk '{ print $2 }' | while read -r n; do + recent_records "Gradle_Test_Executor_$n" + done + echo '```' + fi + samples=$(grep -E '^[0-9]{2}:[0-9]{2}:[0-9]{2} ' "$MEM_LOG" | tail -6) + if [ -n "$samples" ]; then + echo + echo "Last memory samples:" + echo '```text' + echo "$samples" + echo '```' + fi + echo + echo "**Possible causes (inferred, not confirmed)**" + echo + printf '%s' "$hints" + echo + echo "Evidence: artifact \`tron-test-logs-*\` → \`ci-logs/${DIAG_DIR##*/}/\`." + } > "$REPORT" + + [ -n "${GITHUB_STEP_SUMMARY:-}" ] && cat "$REPORT" >> "$GITHUB_STEP_SUMMARY" + # One annotation, shown on the run page and in the PR checks list. + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::${CI_DIAG_ANNOTATION:-error} title=CI diagnosis ($STAGE)::$(printf '%sPossible causes:\n%s' \ + "$observed" "$hints" | sed 's/%/%25/g' | awk '{ printf "%s%%0A", $0 }')" + fi + { + echo "== diagnosis" + printf 'Observed:\n%sPossible causes:\n%s' "$observed" "$hints" + } >> "$STATUS_FILE" + log "diagnosis written to $REPORT" + printf 'Observed:\n%sPossible causes:\n%s' "$observed" "$hints" +} + +# Test progress records: "