From 86d91f9f10a217ba73760fac1dc677e6d778ac18 Mon Sep 17 00:00:00 2001 From: FORGE Date: Sun, 25 Oct 2026 00:57:50 +0800 Subject: [PATCH 1/6] ci: cut motd into eleven segments over three jobs The eight motd segments left two jobs at 98 s and 77 s wall clock and a 63 s slowest segment. The three heaviest segments are split along their case-lifecycle boundaries instead of by line count, giving eleven segments spread over jobs of four, four and three: * segment 4 keeps the target, pointer and pending hook cases, segment 5 takes the six building phases with their three signals; * segment 7 keeps the dependency, noop and final-group cases, segment 8 takes the illegal legacy names, legacy backups and legal imports; * segments 9 and 10 run one implementation of the rollback-damage cases each and keep the normal rollback cases with segment 10; those cases move verbatim into run_rollback_damage_cases, so both halves share one body instead of running it twice. Every segment still builds its own fixtures, and with TEST_SEGMENT unset the guards run in file order, which is the original order. tests/run.sh keeps the plan precheck, the duplicate detection and the child environment cleanup; the segment count is bounded lexically to 1..12 and every TEST_SEGMENT_SET entry to 0..11 before the arithmetic range comparison. tests/test-shards.sh follows the eleven-segment expectations, runs segment 10 as executable evidence for two-digit indices, and validates every segment_set token strictly, so "8 9 10x" is refused instead of being filtered out by the coverage arithmetic. --- .github/workflows/shell-tests.yml | 24 ++++-- tests/run.sh | 10 +-- tests/test-motd.sh | 137 ++++++++++++++++-------------- tests/test-shards.sh | 42 +++++++-- 4 files changed, 129 insertions(+), 84 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index be5a514..0ed59ef 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -59,16 +59,20 @@ jobs: fail-fast: false matrix: include: - # motd and cloudflare are cut into eight segments; each of their two - # jobs runs four of them, so a single suite spans two jobs. + # motd is cut into eleven segments spread over three jobs of four, + # four and three; cloudflare is cut into eight, two jobs of four. - shard: motd-a suite: motd - segments: 8 + segments: 11 segment_set: "0 1 2 3" - shard: motd-b suite: motd - segments: 8 + segments: 11 segment_set: "4 5 6 7" + - shard: motd-c + suite: motd + segments: 11 + segment_set: "8 9 10" - shard: push-worker suite: push-worker segments: 4 @@ -125,16 +129,20 @@ jobs: fail-fast: false matrix: include: - # motd and cloudflare are cut into eight segments; each of their two - # jobs runs four of them, so a single suite spans two jobs. + # motd is cut into eleven segments spread over three jobs of four, + # four and three; cloudflare is cut into eight, two jobs of four. - shard: motd-a suite: motd - segments: 8 + segments: 11 segment_set: "0 1 2 3" - shard: motd-b suite: motd - segments: 8 + segments: 11 segment_set: "4 5 6 7" + - shard: motd-c + suite: motd + segments: 11 + segment_set: "8 9 10" - shard: push-worker suite: push-worker segments: 4 diff --git a/tests/run.sh b/tests/run.sh index b787d72..de77c46 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -71,14 +71,14 @@ done # defines. TEST_SEGMENTS sets how many processes share the work; with # TEST_SEGMENTS=1 every suite runs in one process, all segments in order. declare -rA SEGMENTED_SUITES=( - [test-motd.sh]=8 + [test-motd.sh]=11 [test-cloudflare-tunnel.sh]=8 [test-push-worker-registration.sh]=4 ) segments=${TEST_SEGMENTS:-1} -# One digit, at most the eight segments a suite actually has: no accepted value -# can overflow shell arithmetic and every index below stays a single character. -[[ "$segments" =~ ^[1-8]$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } +# Bounded lexical form first, so no accepted value can overflow shell arithmetic +# or smuggle in a sign, a prefix or a multi-digit index past the range checks. +[[ "$segments" =~ ^([1-9]|1[0-2])$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } # TEST_SEGMENT_SET narrows one job to a subset of the segments ("0 1 2 3"), so a # single suite can be spread over several jobs. Unset or blank keeps every @@ -94,7 +94,7 @@ if [[ -n "${TEST_SEGMENT_SET:-}" && -n "${TEST_SEGMENT_SET//[[:space:]]/}" ]]; t segment_value=${segment_value//$'\n'/ } read -r -a segment_entries <<< "$segment_value" for segment_index in "${segment_entries[@]}"; do - [[ "$segment_index" =~ ^[0-7]$ ]] \ + [[ "$segment_index" =~ ^([0-9]|1[01])$ ]] \ || { printf 'Invalid TEST_SEGMENT_SET entry: %s\n' "$segment_index" >&2; exit 2; } (( segment_index < segments )) \ || { printf 'Segment %s out of range for TEST_SEGMENTS=%s\n' "$segment_index" "$segments" >&2; exit 2; } diff --git a/tests/test-motd.sh b/tests/test-motd.sh index a7ea717..1001493 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -191,6 +191,68 @@ motd_snapshot_capture_hook() { motd_run_locked_operation install CHILD chmod 0700 "$TEST_DIR/building-signal-child.sh" +run_rollback_damage_cases() { + local rollback_impl=$1 rollback_damage root + for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do + ( + root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" + if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install + case "$rollback_damage" in + motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; + absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + before=$(target_hash "$root") + before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + motd_rollback_hook() { + local id="$1" path target + [[ "$id" == motd_news ]] || return 0 + case "$rollback_damage" in + motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) + path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" + cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" + ;; + absent-regular|absent-symlink|absent-fifo) + ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + rm() { + local status + command rm "$@"; status=$?; unset -f rm + case "$rollback_damage" in + absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; + absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; + absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; + esac + return "$status" + } + ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + return 0 + } + assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" + transaction=$(cat "$MOTD_STATE_DIR/pending") + journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" + grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" + [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" + pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" + motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } + motd_take_lock; motd_reconcile_state; motd_release_lock + assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" + assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" + motd_take_lock; motd_release_lock + ) + done +} + if seg_is 0; then extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; @@ -934,8 +996,9 @@ for pending_hook_case in id extra-line inode; do assert_no_motd_residue "$root" ) done +fi - +if seg_is 5; then for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do ( root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" @@ -969,7 +1032,7 @@ for building_phase in transaction-snapshot-building capture-motd capture-issue_n done fi -if seg_is 5; then +if seg_is 6; then for previous_impl in tool module; do ( root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" @@ -1065,7 +1128,7 @@ done ) fi -if seg_is 6; then +if seg_is 7; then for missing_command in bash chown flock ln od readlink sha256sum touch wc; do ( root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" @@ -1138,7 +1201,9 @@ for group_impl in tool module; do ) done done +fi +if seg_is 8; then for illegal_legacy_name in motd.previous-unknown issue.previous-unknown motd.invalid-backup evil.initial-backup; do ( root="$TEST_DIR/legacy-name-${illegal_legacy_name//[^a-zA-Z0-9]/-}"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install @@ -1200,68 +1265,12 @@ done ) fi -if seg_is 7; then -for rollback_impl in tool module; do - for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do - ( - root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" - if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install - case "$rollback_damage" in - motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; - absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - before=$(target_hash "$root") - before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - motd_rollback_hook() { - local id="$1" path target - [[ "$id" == motd_news ]] || return 0 - case "$rollback_damage" in - motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) - path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" - cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" - ;; - absent-regular|absent-symlink|absent-fifo) - ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - rm() { - local status - command rm "$@"; status=$?; unset -f rm - case "$rollback_damage" in - absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; - absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; - absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; - esac - return "$status" - } - ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - return 0 - } - assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" - transaction=$(cat "$MOTD_STATE_DIR/pending") - journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" - grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" - [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" - pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" - motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } - motd_take_lock; motd_reconcile_state; motd_release_lock - assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" - assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" - motd_take_lock; motd_release_lock - ) - done +if seg_is 9; then +run_rollback_damage_cases tool +fi -done +if seg_is 10; then +run_rollback_damage_cases module for rollback_impl in tool module; do ( diff --git a/tests/test-shards.sh b/tests/test-shards.sh index c8bde8d..9bd06f3 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -142,7 +142,8 @@ jobs=$(awk ' # covered exactly once, union 0..N-1. check_matrix() { local job=$1 rows suite row_suite suite_segments segment_set_value - local declared covered index count total expected actual + local declared covered index count total expected actual token + local -a set_tokens=() rows=$(printf '%s\n' "$jobs" | awk -F'|' -v job="$job" '$1 == job') [[ -n "$rows" ]] || fail "no matrix rows for $job" if printf '%s\n' "$rows" | cut -d'|' -f2 | sort | uniq -d | grep -q .; then @@ -156,6 +157,18 @@ check_matrix() { case "$suite_segments" in ''|*[!0-9]*) fail "$job $suite has an unsupported segment count: '$suite_segments'" ;; esac + if [[ -n "$segment_set_value" ]]; then + # Strict tokens, checked before any coverage arithmetic: a set + # carries decimal indices of the declared range, so a trailing + # "x" (or "10x") fails here instead of being filtered out. + read -r -a set_tokens <<< "$segment_set_value" + for token in "${set_tokens[@]}"; do + [[ "$token" =~ ^([0-9]|1[01])$ ]] \ + || fail "$job $suite has an invalid segment_set entry: '$token'" + (( token < suite_segments )) \ + || fail "$job $suite segment_set entry $token is outside 0..$(( suite_segments - 1 ))" + done + fi if [[ -z "$declared" ]]; then declared=$suite_segments elif [[ "$declared" != "$suite_segments" ]]; then @@ -263,18 +276,25 @@ check_subset() { fail "subset '$set_value' for $suite ran: $actual" } } -check_subset motd 8 '0 1 2 3' 'test-motd.sh-0 +check_subset motd 11 '0 1 2 3' 'test-motd.sh-0 test-motd.sh-1 test-motd.sh-2 test-motd.sh-3' -check_subset motd 8 '' 'test-motd.sh-0 +# Double-digit segments must run as themselves, not as "1" plus a stray "0". +check_subset motd 11 '8 9 10' 'test-motd.sh-8 +test-motd.sh-9 +test-motd.sh-10' +check_subset motd 11 '' 'test-motd.sh-0 test-motd.sh-1 test-motd.sh-2 test-motd.sh-3 test-motd.sh-4 test-motd.sh-5 test-motd.sh-6 -test-motd.sh-7' +test-motd.sh-7 +test-motd.sh-8 +test-motd.sh-9 +test-motd.sh-10' check_subset motd 1 '' 'test-motd.sh' check_subset cloudflare 8 '4 5 6 7' 'test-cloudflare-tunnel.sh-4 test-cloudflare-tunnel.sh-5 @@ -299,17 +319,19 @@ check_reject() { # A segment count the suite's source does not have would silently run part of it. check_reject 'motd cut into two' motd 2 '' check_reject 'motd cut into four' motd 4 '' +check_reject 'motd cut into eight' motd 8 '' +check_reject 'motd cut into twelve' motd 12 '' check_reject 'cloudflare cut into four' cloudflare 4 '' check_reject 'push-worker cut into eight' push-worker 8 '' check_reject 'push-worker cut into two' push-worker 2 '' -check_reject 'motd segment past its source' motd 8 '8' +check_reject 'motd segment past its source' motd 11 '11' check_reject 'subset for an unsegmented suite' push 2 '0' check_reject 'subset for the other bucket' other 2 '0' check_reject 'subset without segmented mode' motd 1 '0' check_reject 'mixed plan with one count' all 8 '' check_reject 'mixed plan with four segments' all 4 '0 1 2 3' -# Anything but a distinct single-digit in-range index is refused; a file named 1 -# must not turn "[1]" into a real segment. +# Anything but a distinct in-range index of the declared range is refused; a +# file named 1 must not turn "[1]" into a real segment. : > "$TEMP_DIR/1" for bad in '1 01' '9223372036854775808' '18446744073709551616' '[1]' '1,2' '-1' '1 1' '+1' '0x1' 'x'; do : > "$CAPTURE" @@ -317,6 +339,12 @@ for bad in '1 01' '9223372036854775808' '18446744073709551616' '[1]' '1,2' '-1' ( cd "$TEMP_DIR" && TEST_SEGMENTS=8 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted (exit $rc)"; } done +for bad in '10x' '11' '010' '1 10 10' '0 11'; do + : > "$CAPTURE" + rc=0 + ( cd "$TEMP_DIR" && TEST_SEGMENTS=11 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted at TEST_SEGMENTS=11 (exit $rc)"; } +done for bad in 0 9 08 '4x' '1 4' 9223372036854775808 18446744073709551616; do : > "$CAPTURE" rc=0 From 1bbfcdde857606581949930e498e32fd1cffeae0 Mon Sep 17 00:00:00 2001 From: FORGE Date: Sun, 25 Oct 2026 02:27:13 +0800 Subject: [PATCH 2/6] ci: cut push-worker into seven segments and lint both groups in parallel --- .github/workflows/shell-tests.yml | 39 +- tests/run.sh | 2 +- tests/test-lint-coverage.sh | 78 +++- tests/test-push-worker-registration.sh | 494 +++++++++++++++++-------- tests/test-shards.sh | 46 ++- 5 files changed, 471 insertions(+), 188 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 0ed59ef..002a61e 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -45,10 +45,19 @@ jobs: shell: bash run: | # All tracked Bash scripts outside tests are production by default. - # Check both groups even when production lint fails. + # Both groups lint in parallel, one file per process, and both are + # started before the first wait so a failure in one cannot hide the + # other. pipefail stays on, so a failed file enumeration fails its + # group even though xargs then sees empty input; any failure becomes + # a single exit 1. + set -o pipefail status=0 - git ls-files -z -- "*.sh" ":!:tests/**" | xargs -0 -r shellcheck -x --severity=warning -- || status=1 - git ls-files -z -- "tests/*.sh" | xargs -0 -r shellcheck -x --severity=error -- || status=1 + git ls-files -z -- "*.sh" ":!:tests/**" | xargs -0 -r -n1 -P4 shellcheck -x --severity=warning -- & + production_pid=$! + git ls-files -z -- "tests/*.sh" | xargs -0 -r -n1 -P4 shellcheck -x --severity=error -- & + test_pid=$! + wait "$production_pid" || status=1 + wait "$test_pid" || status=1 exit "$status" ubuntu-tests: @@ -60,7 +69,8 @@ jobs: matrix: include: # motd is cut into eleven segments spread over three jobs of four, - # four and three; cloudflare is cut into eight, two jobs of four. + # four and three; push-worker is cut into seven, two jobs of four and + # three; cloudflare is cut into eight, two jobs of four. - shard: motd-a suite: motd segments: 11 @@ -73,9 +83,14 @@ jobs: suite: motd segments: 11 segment_set: "8 9 10" - - shard: push-worker + - shard: push-worker-a suite: push-worker - segments: 4 + segments: 7 + segment_set: "0 1 2 3" + - shard: push-worker-b + suite: push-worker + segments: 7 + segment_set: "4 5 6" - shard: push suite: push segments: 1 @@ -130,7 +145,8 @@ jobs: matrix: include: # motd is cut into eleven segments spread over three jobs of four, - # four and three; cloudflare is cut into eight, two jobs of four. + # four and three; push-worker is cut into seven, two jobs of four and + # three; cloudflare is cut into eight, two jobs of four. - shard: motd-a suite: motd segments: 11 @@ -143,9 +159,14 @@ jobs: suite: motd segments: 11 segment_set: "8 9 10" - - shard: push-worker + - shard: push-worker-a + suite: push-worker + segments: 7 + segment_set: "0 1 2 3" + - shard: push-worker-b suite: push-worker - segments: 4 + segments: 7 + segment_set: "4 5 6" - shard: push suite: push segments: 1 diff --git a/tests/run.sh b/tests/run.sh index de77c46..1acf97a 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -73,7 +73,7 @@ done declare -rA SEGMENTED_SUITES=( [test-motd.sh]=11 [test-cloudflare-tunnel.sh]=8 - [test-push-worker-registration.sh]=4 + [test-push-worker-registration.sh]=7 ) segments=${TEST_SEGMENTS:-1} # Bounded lexical form first, so no accepted value can overflow shell arithmetic diff --git a/tests/test-lint-coverage.sh b/tests/test-lint-coverage.sh index bb14d00..e210652 100644 --- a/tests/test-lint-coverage.sh +++ b/tests/test-lint-coverage.sh @@ -16,14 +16,20 @@ awk ' block { step=0; block=0 } END { if (found != 1 || lines == 0) exit 1 } ' "$ROOT_DIR/.github/workflows/shell-tests.yml" > "$root/lint.sh" +# Both groups must keep the per-file, four-way parallel invocation. +if [[ "$(grep -c -- '-n1 -P4' "$root/lint.sh")" != 2 ]]; then + printf 'the lint block no longer starts both groups with -n1 -P4\n' >&2 + exit 1 +fi mkdir -p "$root/bin" "$root/repo" cat > "$root/bin/shellcheck" <<'STUB' #!/usr/bin/env bash set -euo pipefail -[[ $1 == -x && $3 == -- && $# -gt 3 ]] || exit 90 +# One file per call (xargs -n1), classified by severity: anything else is a defect. +[[ $# -eq 4 && $1 == -x && $3 == -- ]] || exit 90 level=$2 shift 3 -for path; do printf "%s\t%s\n" "$level" "$path" >> "$CAPTURE"; done +printf "%s\t%s\n" "$level" "$1" >> "$CAPTURE" case $level in --severity=warning) exit "$PRODUCTION_STATUS" ;; --severity=error) exit "$TEST_STATUS" ;; @@ -31,6 +37,20 @@ case $level in esac STUB chmod +x "$root/bin/shellcheck" +real_git=$(command -v git) +cat > "$root/bin/git" < "$root/production.expected" +printf "%s\n" \ $'--severity=error\ttests/direct.sh' \ - $'--severity=error\ttests/nested/deep.sh' > "$root/expected" + $'--severity=error\ttests/nested/deep.sh' > "$root/tests.expected" +cat "$root/production.expected" "$root/tests.expected" > "$root/all.expected" +# Multiset comparison: every tracked script of both groups exactly once, with +# its own group's severity. Sorting keeps the parallel order out of the verdict, +# while a duplicated or a missing file still shows up as a difference. +check_linted() { + local label=$1 expected=$2 + LC_ALL=C sort "$root/capture" > "$root/capture.sorted" + LC_ALL=C sort "$expected" > "$root/expected.sorted" + if ! diff -u "$root/expected.sorted" "$root/capture.sorted"; then + printf 'FAIL: %s: linted file set is not the tracked split\n' "$label" >&2 + exit 1 + fi +} +run_lint() { + local status=0 + bash --noprofile --norc -e -o pipefail "$root/lint.sh" || status=$? + printf '%s' "$status" +} +# Both groups must always be linted in full, and a failure in either one must +# fail the step: the four result combinations cover that. +export ENUM_FAIL=none for PRODUCTION_STATUS in 0 1; do for TEST_STATUS in 0 1; do export PRODUCTION_STATUS TEST_STATUS : > "$CAPTURE" - status=0 - bash --noprofile --norc -e -o pipefail "$root/lint.sh" || status=$? + status=$(run_lint) expected=0 if (( PRODUCTION_STATUS != 0 || TEST_STATUS != 0 )); then expected=1; fi - [[ $status == "$expected" ]] - diff -u "$root/expected" "$CAPTURE" - printf "PASS: workflow lint production=%s tests=%s exit=%s\n" "$PRODUCTION_STATUS" "$TEST_STATUS" "$status" + [[ $status == "$expected" ]] \ + || { printf 'FAIL: lint exited %s, expected %s\n' "$status" "$expected" >&2; exit 1; } + check_linted "production=$PRODUCTION_STATUS tests=$TEST_STATUS" "$root/all.expected" + printf "PASS: workflow lint production=%s tests=%s exit=%s\n" \ + "$PRODUCTION_STATUS" "$TEST_STATUS" "$status" done done +# A failed enumeration must fail the step without hiding the other group: the +# background pipeline keeps pipefail, so git failing is not swallowed by the +# empty xargs run behind it. +for ENUM_FAIL in production tests both; do + export ENUM_FAIL PRODUCTION_STATUS=0 TEST_STATUS=0 + : > "$CAPTURE" + status=$(run_lint) + [[ $status == 1 ]] \ + || { printf 'FAIL: %s enumeration failure exited %s\n' "$ENUM_FAIL" "$status" >&2; exit 1; } + case "$ENUM_FAIL" in + production) check_linted "$ENUM_FAIL enumeration failure" "$root/tests.expected" ;; + tests) check_linted "$ENUM_FAIL enumeration failure" "$root/production.expected" ;; + both) [[ ! -s "$CAPTURE" ]] \ + || { printf 'FAIL: both enumerations failed but files were linted\n' >&2; exit 1; } ;; + esac + printf 'PASS: %s enumeration failure exits 1\n' "$ENUM_FAIL" +done diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh index acb194e..99da562 100755 --- a/tests/test-push-worker-registration.sh +++ b/tests/test-push-worker-registration.sh @@ -214,6 +214,187 @@ EOF chmod 0700 "$root/bin"/* } +# Two case bodies below are long enough to be shared by a pair of segments: the +# retry stress and the failed-reap runs. Each body lives here once and both +# segments run a contiguous half of the runs, so no case body is duplicated and +# no PASS summary is repeated. Only the half holding the last run prints the +# total, which keeps the summary text identical to the unsegmented suite. +run_preserved_cleanup_retry_case() ( + local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file + local owner watchdog rc=0 metadata="" state_status=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + credential="$runtime/private-key" + printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" + RUNTIME_KEY_FILE=$credential + + setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do + read_process_record "$leader" || true + [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break + sleep 0.01 + done + read_process_record "$leader" + leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start + WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid + publish_worker_session_state active + state_file=$WORKER_SESSION_STATE_FILE + + eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" + terminate_managed_session() { return 1; } + owner=$BASHPID + test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! + cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + + (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" + [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" + [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || + fail "retry run $run lost runtime identity" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + metadata=$(stat -Lc '%u:%g:%a' -- "$credential") + [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" + grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" + state_status=0; read_worker_session_state "$state_file" || state_status=$? + assert_eq 0 "$state_status" "retry run $run retained state parses" + [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && + "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || + fail "retry run $run retained state identity mismatch" + [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && + "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" + test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" + read_process_record "$leader" + [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" + + eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" + cleanup_runtime + wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" + [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || + fail "retry run $run left state or runtime" + runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" + [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" + pass "failed cleanup evidence retry run $run preserves then clears runtime" +) + +runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} +[[ "$runtime_evidence_stress" =~ ^[0-9]+$ ]] \ + || fail "invalid PUSH_RUNTIME_EVIDENCE_STRESS: $runtime_evidence_stress" +# The ceiling keeps an odd count working: one half always holds the last run. +runtime_evidence_split=$(( (runtime_evidence_stress + 1) / 2 )) + +# The failed-reap child and case body are shared for the same reason: part one +# of segment 0 costs about 18 s while each seeded run costs about 2.8 s, so the +# twenty runs split 8/12 and both segments stay inside the time budget. +cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" mode="$3" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime +setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! +for _ in {1..100}; do + leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) + leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) + leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) + [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break + sleep 0.01 +done +[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 +printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" +WORKER_TRANSFER_PID=$leader +WORKER_TRANSFER_START=$leader_start +WORKER_TRANSFER_PGID=$leader_pgid +WORKER_TRANSFER_SID=$leader_sid +publish_worker_session_state active +state_file=$WORKER_SESSION_STATE_FILE +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { return 1; } +start_ns=$(date +%s%N) +rc=0 +terminate_worker_transfer || rc=$? +elapsed_ns=$(( $(date +%s%N) - start_ns )) +[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 +[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 +read_worker_session_state "$state_file" +[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 +next_rc=0 +run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? +[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 +printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" +: > "$root/capture/returned" +if [[ "$mode" == hold ]]; then + while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done + for _ in {1..400}; do + reap_status=0 + reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? + case "$reap_status" in + 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; + *) sleep 0.01 ;; + esac + done + (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 + : > "$root/capture/reaped" + for _ in {1..400}; do + [[ ! -e "$state_file" && ! -L "$state_file" ]] && break + sleep 0.01 + done + [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 + terminate_managed_session() { original_terminate_managed_session "$@"; } +else + for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done + [[ -e "$root/capture/release-recovery" ]] || exit 97 + terminate_managed_session() { original_terminate_managed_session "$@"; } +fi +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/failed-reap-child.sh" + +run_failed_reap_case() { + local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 + bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! + test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! + for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" + : > "$root/capture/release-recovery" + wait "$child" || rc=$? + [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true + assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" + [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" +} + +# The state-publication matrix is shared by segments 5 and 6 the same way: the +# combinations keep their original state, phase and signal order and split into +# two contiguous halves, so overriding the variables below still puts every +# combination in exactly one half. +publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} +publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} +publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} +publication_total=0 +for publication_state in $publication_states; do + for publication_phase in $publication_phases; do + for publication_signal in $publication_signals; do + publication_total=$(( publication_total + 1 )) + done + done +done +publication_split=$(( (publication_total + 1) / 2 )) + if seg_is 0; then source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' @@ -576,89 +757,23 @@ done pass "managed leader PID reuse is treated as original session gone" ) -cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" mode="$3" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime -setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! -for _ in {1..100}; do - leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) - leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) - leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) - [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break - sleep 0.01 -done -[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 -printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" -WORKER_TRANSFER_PID=$leader -WORKER_TRANSFER_START=$leader_start -WORKER_TRANSFER_PGID=$leader_pgid -WORKER_TRANSFER_SID=$leader_sid -publish_worker_session_state active -state_file=$WORKER_SESSION_STATE_FILE -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { return 1; } -start_ns=$(date +%s%N) -rc=0 -terminate_worker_transfer || rc=$? -elapsed_ns=$(( $(date +%s%N) - start_ns )) -[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 -[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 -read_worker_session_state "$state_file" -[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 -next_rc=0 -run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? -[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 -printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" -: > "$root/capture/returned" -if [[ "$mode" == hold ]]; then - while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done - for _ in {1..400}; do - reap_status=0 - reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? - case "$reap_status" in - 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; - *) sleep 0.01 ;; - esac - done - (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 - : > "$root/capture/reaped" - for _ in {1..400}; do - [[ ! -e "$state_file" && ! -L "$state_file" ]] && break - sleep 0.01 - done - [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 - terminate_managed_session() { original_terminate_managed_session "$@"; } -else - for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done - [[ -e "$root/capture/release-recovery" ]] || exit 97 - terminate_managed_session() { original_terminate_managed_session "$@"; } +# First half of the twenty failed-reap runs. Segment 1 starts at run 9 and +# holds the summary, so the runs keep their original order and the summary +# still appears exactly once. +for failed_reap_run in $(seq 1 8); do run_failed_reap_case "$failed_reap_run"; done fi -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/failed-reap-child.sh" -run_failed_reap_case() { - local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 - bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! - test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! - for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" - : > "$root/capture/release-recovery" - wait "$child" || rc=$? - [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true - assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" - [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" -} -for failed_reap_run in $(seq 1 20); do run_failed_reap_case "$failed_reap_run"; done +if seg_is 1; then +# PUSH_REAP_ONLY keeps its original meaning for the whole suite: with it set, +# nothing after the deadline case runs. That used to be one exit inside +# segment 0, so the second half carries the same guard. +if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then + exit 0 +fi + +# Second half of the twenty failed-reap runs, then the parent fallback that +# follows them in the original order. +for failed_reap_run in $(seq 9 20); do run_failed_reap_case "$failed_reap_run"; done pass "failed managed-session reap remains bounded for 20 deterministic runs" ( @@ -692,7 +807,7 @@ pass "failed managed-session reap remains bounded for 20 deterministic runs" ) fi -if seg_is 1; then +if seg_is 2; then ( trap - EXIT HUP INT TERM sleep 60 & replacement=$! @@ -780,81 +895,27 @@ chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" pass "hard timeout cleanup failure is bounded and blocks subsequent command" ) -run_preserved_cleanup_retry_case() ( - local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file - local owner watchdog rc=0 metadata="" state_status=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - credential="$runtime/private-key" - printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" - RUNTIME_KEY_FILE=$credential - - setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do - read_process_record "$leader" || true - [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break - sleep 0.01 - done - read_process_record "$leader" - leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start - WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid - publish_worker_session_state active - state_file=$WORKER_SESSION_STATE_FILE - - eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" - terminate_managed_session() { return 1; } - owner=$BASHPID - test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! - cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - - (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" - [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" - [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || - fail "retry run $run lost runtime identity" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - metadata=$(stat -Lc '%u:%g:%a' -- "$credential") - [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" - grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" - state_status=0; read_worker_session_state "$state_file" || state_status=$? - assert_eq 0 "$state_status" "retry run $run retained state parses" - [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && - "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || - fail "retry run $run retained state identity mismatch" - [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && - "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" - test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" - read_process_record "$leader" - [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" - - eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" - cleanup_runtime - wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" - [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || - fail "retry run $run left state or runtime" - runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" - [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" - pass "failed cleanup evidence retry run $run preserves then clears runtime" -) +# First half of the shared retry stress; the summary stays with the last run, +# so a lower PUSH_RUNTIME_EVIDENCE_STRESS still reports its total once. +for runtime_evidence_run in $(seq 1 "$runtime_evidence_split"); do + run_preserved_cleanup_retry_case "$runtime_evidence_run" +done +if (( runtime_evidence_split >= runtime_evidence_stress )); then + pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +fi +fi -runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} -for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do +if seg_is 3; then +# Second half of the shared retry stress. +for runtime_evidence_run in $(seq $(( runtime_evidence_split + 1 )) "$runtime_evidence_stress"); do run_preserved_cleanup_retry_case "$runtime_evidence_run" done -pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +if (( runtime_evidence_split < runtime_evidence_stress )); then + pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +fi fi -if seg_is 2; then +if seg_is 4; then ( local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE @@ -1338,7 +1399,7 @@ run_active_grace_signal_case INT 130 run_active_grace_signal_case TERM 143 fi -if seg_is 3; then +if seg_is 5; then cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1413,6 +1474,8 @@ run_live_cleanup_failure_signal_case HUP 129 run_live_cleanup_failure_signal_case INT 130 run_live_cleanup_failure_signal_case TERM 143 +# Both publication halves carry their own child fixture and case runner: the +# two halves are separate processes, so neither may rely on the other. cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1510,12 +1573,130 @@ run_state_publication_signal_case() ( trap - EXIT pass "$state/$phase/$signal_name publication window cleans process tree and runtime" ) -publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} -publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} -publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} + +# First half of the shared publication matrix, in the original state, phase and +# signal order. +publication_index=0 for publication_state in $publication_states; do for publication_phase in $publication_phases; do for publication_signal in $publication_signals; do + publication_index=$(( publication_index + 1 )) + (( publication_index <= publication_split )) || break 3 + case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac + run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" + done + done +done +fi + +if seg_is 6; then +# Both publication halves carry their own child fixture and case runner: the +# two halves are separate processes, so neither may rely on the other. +cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +main_owner=$BASHPID +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$BASHPID" != "$main_owner" ]]; then return 1; fi + original_terminate_managed_session "$@" +} +worker_session_state_publish_hook() { + local phase="$1" state="$2" state_file="$3" hook_stage="$4" marker_stage worker_start + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$state" == active && "$phase" == state-file-assigned ]]; then + : > "$STATE_CLEANUP_RELEASE" + fi + [[ "$phase" == "$STATE_PUBLISH_PHASE" && "$state" == "$STATE_PUBLISH_STATE" ]] || return 0 + [[ ! -e "$STATE_PUBLISH_MARKER" ]] || return 0 + worker_start="" + for _ in {1..20}; do + if read_process_record "$BASHPID" && [[ "$PROC_UID" == "$EUID" && "$PROC_STATE" != Z ]]; then worker_start=$PROC_START; break; fi + sleep 0.01 + done + [[ -n "$worker_start" ]] || return 1 + marker_stage="$STATE_PUBLISH_MARKER.$BASHPID.stage" + (umask 077; set -o noclobber; : > "$marker_stage") || return 1 + printf 'phase=%s\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\nstate_file=%s\nhook_stage=%s\n' \ + "$phase" "$state" "$BASHPID" "$worker_start" "$WORKER_TRANSFER_PID" "$WORKER_TRANSFER_START" \ + "$WORKER_TRANSFER_PGID" "$WORKER_TRANSFER_SID" "$state_file" "$hook_stage" > "$marker_stage" + chmod 0600 "$marker_stage"; mv -fT "$marker_stage" "$STATE_PUBLISH_MARKER" + while :; do :; done +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/state-publication-child.sh" + +run_state_publication_signal_case() ( + local state="$1" phase="$2" signal_name="$3" expected="$4" mode root main_pid="" main_start="" worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" + local watchdog="" watchdog_start="" unrelated="" unrelated_start="" rc=0 line key value marker state_file hook_stage + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + cleanup_publication_fixture() { + [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } + [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } + [[ -z "$watchdog" || -z "$watchdog_start" ]] || { test_process_identity_exists "$watchdog" "$watchdog_start" && kill -TERM "$watchdog" 2>/dev/null || true; } + [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } + [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true + [[ -z "$watchdog" ]] || wait "$watchdog" 2>/dev/null || true + [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true + } + trap cleanup_publication_fixture EXIT + mode=active; [[ "$state" == cleanup_failed ]] && mode=cleanup + root="$TEST_DIR/state-publish-$state-$phase-$signal_name${5:-}"; CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + mkdir -m 0700 "$root/capture" + marker="$root/capture/publish-marker" + sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + STATE_PUBLISH_MODE="$mode" STATE_PUBLISH_STATE="$state" STATE_PUBLISH_PHASE="$phase" STATE_PUBLISH_MARKER="$marker" \ + STATE_CLEANUP_RELEASE="$root/capture/cleanup-release" bash "$TEST_DIR/state-publication-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$!; main_start=$(wait_test_process_start "$main_pid") + for _ in {1..600}; do [[ -f "$marker" ]] && break; sleep 0.05; done + [[ -f "$marker" && ! -L "$marker" && "$(stat -Lc %a "$marker")" == 600 ]] || { cat "$root/output.log" >&2 || true; fail "$state/$phase/$signal_name did not reach publish hook"; } + while IFS= read -r line; do key=${line%%=*}; value=${line#*=}; case "$key" in worker_pid) worker_pid=$value;;worker_start) worker_start=$value;;leader_pid) leader_pid=$value;;leader_start) leader_start=$value;;sid) managed_sid=$value;;state_file) state_file=$value;;hook_stage) hook_stage=$value;;esac; done < "$marker" + [[ "$worker_pid" =~ ^[1-9][0-9]*$ && "$worker_start" =~ ^[1-9][0-9]*$ && "$managed_sid" =~ ^[1-9][0-9]*$ ]] || fail "$state/$phase/$signal_name marker identity malformed" + wait_test_process_identity_present "$worker_pid" "$worker_start" || { cat "$marker" >&2; fail "$state/$phase/$signal_name worker identity missing at hook"; } + test_watchdog_process 30 "$root/watchdog-timeout" "$main_pid" & watchdog=$!; watchdog_start=$(wait_test_process_start "$watchdog") + printf "DIAG: publication state=%s phase=%s signal=%s main=%s/%s worker=%s/%s sid=%s\n" \ + "$state" "$phase" "$signal_name" "$main_pid" "$main_start" "$worker_pid" "$worker_start" "$managed_sid" + kill "-$signal_name" "$main_pid"; wait "$main_pid" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-timeout" ]] || fail "$state/$phase/$signal_name watchdog fired" + assert_eq "$expected" "$rc" "$state/$phase preserves $signal_name status" + wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$state/$phase/$signal_name left worker identity" + if [[ "$leader_pid" =~ ^[1-9][0-9]*$ && "$leader_start" =~ ^[1-9][0-9]*$ ]]; then + if ! wait_test_process_identity_gone "$leader_pid" "$leader_start"; then + read_process_record "$leader_pid" || true + printf 'leader remains pid=%s expected_start=%s actual_start=%s state=%s ppid=%s pgid=%s sid=%s\n' \ + "$leader_pid" "$leader_start" "${PROC_START:-unknown}" "${PROC_STATE:-unknown}" "${PROC_PPID:-unknown}" "${PROC_PGID:-unknown}" "${PROC_SID:-unknown}" >&2 + cat "/proc/$leader_pid/stat" >&2 2>/dev/null || true + fail "$state/$phase/$signal_name left leader identity" + fi + fi + collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$state/$phase/$signal_name cannot verify managed SID" + (( ${#session_pids[@]} == 0 )) || fail "$state/$phase/$signal_name left managed SID member: ${session_pids[*]}" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$state/$phase/$signal_name left runtime/session/registration state" + [[ ! -e "$state_file" && ! -e "$hook_stage" && ! -L "$state_file" && ! -L "$hook_stage" ]] || fail "$state/$phase/$signal_name left publication path" + test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$state/$phase/$signal_name killed unrelated process" + kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true + unrelated=""; unrelated_start=""; main_pid=""; main_start=""; watchdog=""; watchdog_start="" + trap - EXIT + pass "$state/$phase/$signal_name publication window cleans process tree and runtime" +) + +# Second half of the shared publication matrix. +publication_index=0 +for publication_state in $publication_states; do + for publication_phase in $publication_phases; do + for publication_signal in $publication_signals; do + publication_index=$(( publication_index + 1 )) + (( publication_index > publication_split )) || continue case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" done @@ -1568,4 +1749,3 @@ done printf 'All push worker registration tests passed.\n' fi - diff --git a/tests/test-shards.sh b/tests/test-shards.sh index 9bd06f3..e008397 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -301,11 +301,21 @@ test-cloudflare-tunnel.sh-5 test-cloudflare-tunnel.sh-6 test-cloudflare-tunnel.sh-7' check_subset cloudflare 1 '' 'test-cloudflare-tunnel.sh' -check_subset push-worker 4 '2' 'test-push-worker-registration.sh-2' -check_subset push-worker 4 '' 'test-push-worker-registration.sh-0 +check_subset push-worker 7 '2' 'test-push-worker-registration.sh-2' +check_subset push-worker 7 '0 1 2 3' 'test-push-worker-registration.sh-0 test-push-worker-registration.sh-1 test-push-worker-registration.sh-2 test-push-worker-registration.sh-3' +check_subset push-worker 7 '4 5 6' 'test-push-worker-registration.sh-4 +test-push-worker-registration.sh-5 +test-push-worker-registration.sh-6' +check_subset push-worker 7 '' 'test-push-worker-registration.sh-0 +test-push-worker-registration.sh-1 +test-push-worker-registration.sh-2 +test-push-worker-registration.sh-3 +test-push-worker-registration.sh-4 +test-push-worker-registration.sh-5 +test-push-worker-registration.sh-6' check_subset push-worker 1 '' 'test-push-worker-registration.sh' check_reject() { @@ -323,6 +333,7 @@ check_reject 'motd cut into eight' motd 8 '' check_reject 'motd cut into twelve' motd 12 '' check_reject 'cloudflare cut into four' cloudflare 4 '' check_reject 'push-worker cut into eight' push-worker 8 '' +check_reject 'push-worker cut into four' push-worker 4 '' check_reject 'push-worker cut into two' push-worker 2 '' check_reject 'motd segment past its source' motd 11 '11' check_reject 'subset for an unsegmented suite' push 2 '0' @@ -330,20 +341,31 @@ check_reject 'subset for the other bucket' other 2 '0' check_reject 'subset without segmented mode' motd 1 '0' check_reject 'mixed plan with one count' all 8 '' check_reject 'mixed plan with four segments' all 4 '0 1 2 3' -# Anything but a distinct in-range index of the declared range is refused; a -# file named 1 must not turn "[1]" into a real segment. +# Anything but a distinct in-range index of the declared range is refused, and +# the refusal has to come from the segment validation itself: the declared count +# stays untouched, so a count mismatch cannot stand in for a token that got +# through. A file named 1 must not turn "[1]" into a real segment. +segment_rejection='Invalid TEST_SEGMENT_SET entry|Duplicate segment in TEST_SEGMENT_SET|out of range for TEST_SEGMENTS' +check_bad_set() { + local suite=$1 segs=$2 bad=$3 rc=0 + : > "$CAPTURE" + ( cd "$TEMP_DIR" && TEST_SEGMENTS="$segs" TEST_SEGMENT_SET="$bad" \ + bash "$TEMP_DIR/tests/run.sh" "$suite" > "$TEMP_DIR/output" 2>&1 ) || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' for $suite accepted (exit $rc)"; } + grep -qE "$segment_rejection" "$TEMP_DIR/output" \ + || { cat "$TEMP_DIR/output"; fail "subset '$bad' for $suite was not refused by the segment validation"; } +} : > "$TEMP_DIR/1" for bad in '1 01' '9223372036854775808' '18446744073709551616' '[1]' '1,2' '-1' '1 1' '+1' '0x1' 'x'; do - : > "$CAPTURE" - rc=0 - ( cd "$TEMP_DIR" && TEST_SEGMENTS=8 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? - [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted (exit $rc)"; } + check_bad_set motd 11 "$bad" done for bad in '10x' '11' '010' '1 10 10' '0 11'; do - : > "$CAPTURE" - rc=0 - ( cd "$TEMP_DIR" && TEST_SEGMENTS=11 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? - [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted at TEST_SEGMENTS=11 (exit $rc)"; } + check_bad_set motd 11 "$bad" +done +# The seven-segment suite validates its own tokens the same way, so a set that +# only matches a stale segment count cannot pass for a good one. +for bad in '0 1 2 3x' '4 5 6x' '7' '6 6' '+1' '010'; do + check_bad_set push-worker 7 "$bad" done for bad in 0 9 08 '4x' '1 4' 9223372036854775808 18446744073709551616; do : > "$CAPTURE" From 6d481a938ccb372ebd74a7a95ef68d9ff73e247c Mon Sep 17 00:00:00 2001 From: LUNO Date: Sun, 25 Oct 2026 02:37:11 +0800 Subject: [PATCH 3/6] test(push-worker): interpret stress override as decimal --- tests/test-push-worker-registration.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh index 99da562..055ffc3 100755 --- a/tests/test-push-worker-registration.sh +++ b/tests/test-push-worker-registration.sh @@ -290,7 +290,7 @@ runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} [[ "$runtime_evidence_stress" =~ ^[0-9]+$ ]] \ || fail "invalid PUSH_RUNTIME_EVIDENCE_STRESS: $runtime_evidence_stress" # The ceiling keeps an odd count working: one half always holds the last run. -runtime_evidence_split=$(( (runtime_evidence_stress + 1) / 2 )) +runtime_evidence_split=$(( (10#$runtime_evidence_stress + 1) / 2 )) # The failed-reap child and case body are shared for the same reason: part one # of segment 0 costs about 18 s while each seeded run costs about 2.8 s, so the @@ -900,7 +900,7 @@ chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" for runtime_evidence_run in $(seq 1 "$runtime_evidence_split"); do run_preserved_cleanup_retry_case "$runtime_evidence_run" done -if (( runtime_evidence_split >= runtime_evidence_stress )); then +if (( runtime_evidence_split >= 10#$runtime_evidence_stress )); then pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" fi fi @@ -910,7 +910,7 @@ if seg_is 3; then for runtime_evidence_run in $(seq $(( runtime_evidence_split + 1 )) "$runtime_evidence_stress"); do run_preserved_cleanup_retry_case "$runtime_evidence_run" done -if (( runtime_evidence_split < runtime_evidence_stress )); then +if (( runtime_evidence_split < 10#$runtime_evidence_stress )); then pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" fi fi From 2f504108f33867560db4835863dcf476bf09d4e0 Mon Sep 17 00:00:00 2001 From: FORGE Date: Sun, 25 Oct 2026 03:17:57 +0800 Subject: [PATCH 4/6] ci: cut cloudflare into twelve segments over three jobs The eight-segment split left the second job at 83 s because it carried the three heaviest segments (30/61/61/66 s) while the first finished in 53 s. Cut the heavy tail instead of piling it up: - the uninstall signal matrix (39 independent cases) splits by signal into segments 3 (HUP, INT) and 4 (TERM), which keeps the serial order HUP, INT, TERM; - the metadata capture matrix becomes segment 7 and the anchor plus creation/publication matrix becomes segment 8; - the last segment (bookkeeping, capture boundaries, restore publishing, disable and uninstall barriers) becomes segments 9, 10 and 11. Every segment still builds its own fixture, serial mode keeps the original order, and the PASS tally is unchanged at 312. Both OS matrices now run four, four and four segments in three jobs whose sets mix early and late segments so the 58 s straggler never shares a job with another heavy half. --- .github/workflows/shell-tests.yml | 32 ++++++++++++++++++++---------- tests/run.sh | 2 +- tests/test-cloudflare-tunnel.sh | 33 +++++++++++++++++++++++++------ tests/test-shards.sh | 32 ++++++++++++++++++++++++++++-- 4 files changed, 80 insertions(+), 19 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 002a61e..ae086d3 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -70,7 +70,9 @@ jobs: include: # motd is cut into eleven segments spread over three jobs of four, # four and three; push-worker is cut into seven, two jobs of four and - # three; cloudflare is cut into eight, two jobs of four. + # three; cloudflare is cut into twelve, three jobs of four whose + # segment sets mix early and late segments so the slowest one is + # never paired with the other heavy halves. - shard: motd-a suite: motd segments: 11 @@ -96,12 +98,16 @@ jobs: segments: 1 - shard: cloudflare-a suite: cloudflare - segments: 8 - segment_set: "0 1 2 3" + segments: 12 + segment_set: "0 2 6 9" - shard: cloudflare-b suite: cloudflare - segments: 8 - segment_set: "4 5 6 7" + segments: 12 + segment_set: "1 3 5 7" + - shard: cloudflare-c + suite: cloudflare + segments: 12 + segment_set: "4 8 10 11" - shard: other suite: other segments: 1 @@ -146,7 +152,9 @@ jobs: include: # motd is cut into eleven segments spread over three jobs of four, # four and three; push-worker is cut into seven, two jobs of four and - # three; cloudflare is cut into eight, two jobs of four. + # three; cloudflare is cut into twelve, three jobs of four whose + # segment sets mix early and late segments so the slowest one is + # never paired with the other heavy halves. - shard: motd-a suite: motd segments: 11 @@ -172,12 +180,16 @@ jobs: segments: 1 - shard: cloudflare-a suite: cloudflare - segments: 8 - segment_set: "0 1 2 3" + segments: 12 + segment_set: "0 2 6 9" - shard: cloudflare-b suite: cloudflare - segments: 8 - segment_set: "4 5 6 7" + segments: 12 + segment_set: "1 3 5 7" + - shard: cloudflare-c + suite: cloudflare + segments: 12 + segment_set: "4 8 10 11" - shard: other suite: other segments: 1 diff --git a/tests/run.sh b/tests/run.sh index 1acf97a..a15213c 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -72,7 +72,7 @@ done # TEST_SEGMENTS=1 every suite runs in one process, all segments in order. declare -rA SEGMENTED_SUITES=( [test-motd.sh]=11 - [test-cloudflare-tunnel.sh]=8 + [test-cloudflare-tunnel.sh]=12 [test-push-worker-registration.sh]=7 ) segments=${TEST_SEGMENTS:-1} diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh index 2e48726..a11b519 100644 --- a/tests/test-cloudflare-tunnel.sh +++ b/tests/test-cloudflare-tunnel.sh @@ -507,8 +507,11 @@ find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -qu pass "configure then real repository uninstall removes source and preserves keyring" fi -if seg_is 3; then -for signal in HUP INT TERM; do +# The uninstall signal matrix is 39 independent cases (3 signals x 13 phases), +# each building its own fixture. Segments 3 and 4 split it by signal, so the +# serial order stays HUP, INT, TERM exactly as before. +uninstall_signal_phase_cases() { + local signal=$1 phase case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do new_case @@ -537,10 +540,18 @@ for signal in HUP INT TERM; do release_repository_lock || fail "uninstall $signal at $phase lock cleanup failed" pass "uninstall $signal=$expected cleans $phase" done -done +} + +if seg_is 3; then +uninstall_signal_phase_cases HUP +uninstall_signal_phase_cases INT fi if seg_is 4; then +uninstall_signal_phase_cases TERM +fi + +if seg_is 5; then new_case set_old_generation configure_repository @@ -667,7 +678,7 @@ for schema_case in version generation duplicate unknown order state mode uid gid done fi -if seg_is 5; then +if seg_is 6; then for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do ( new_case @@ -966,7 +977,7 @@ for restore_id in auto_update_script auto_update_service auto_update_timer sourc done fi -if seg_is 6; then +if seg_is 7; then for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do for bad_metadata in mode owner gid symlink directory fifo; do ( @@ -998,6 +1009,9 @@ for target_id in auto_update_script auto_update_service auto_update_timer source pass "capture rejects $target_id $bad_metadata without deleting target" done done +fi + +if seg_is 8; then ( unset CLOUDFLARED_TRUST_ANCHOR @@ -1049,7 +1063,7 @@ for creation_phase in before-create-marker before-create-manifest-stage before-c done fi -if seg_is 7; then +if seg_is 9; then for bookkeeping_fault in marker stage rename; do ( new_case @@ -1148,6 +1162,9 @@ for capture_file in old-key old-source old-current; do pass "repository capture $capture_file $capture_fault preserves formal bytes and metadata" done done +fi + +if seg_is 10; then ( new_case @@ -1256,6 +1273,10 @@ for restore_object in key source current; do done done +fi + +if seg_is 11; then + for operation in install_cloudflared upgrade_cloudflared; do ( new_case diff --git a/tests/test-shards.sh b/tests/test-shards.sh index e008397..95be616 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -296,10 +296,30 @@ test-motd.sh-8 test-motd.sh-9 test-motd.sh-10' check_subset motd 1 '' 'test-motd.sh' -check_subset cloudflare 8 '4 5 6 7' 'test-cloudflare-tunnel.sh-4 -test-cloudflare-tunnel.sh-5 +check_subset cloudflare 12 '0 2 6 9' 'test-cloudflare-tunnel.sh-0 +test-cloudflare-tunnel.sh-2 test-cloudflare-tunnel.sh-6 +test-cloudflare-tunnel.sh-9' +check_subset cloudflare 12 '1 3 5 7' 'test-cloudflare-tunnel.sh-1 +test-cloudflare-tunnel.sh-3 +test-cloudflare-tunnel.sh-5 test-cloudflare-tunnel.sh-7' +check_subset cloudflare 12 '4 8 10 11' 'test-cloudflare-tunnel.sh-4 +test-cloudflare-tunnel.sh-8 +test-cloudflare-tunnel.sh-10 +test-cloudflare-tunnel.sh-11' +check_subset cloudflare 12 '' 'test-cloudflare-tunnel.sh-0 +test-cloudflare-tunnel.sh-1 +test-cloudflare-tunnel.sh-2 +test-cloudflare-tunnel.sh-3 +test-cloudflare-tunnel.sh-4 +test-cloudflare-tunnel.sh-5 +test-cloudflare-tunnel.sh-6 +test-cloudflare-tunnel.sh-7 +test-cloudflare-tunnel.sh-8 +test-cloudflare-tunnel.sh-9 +test-cloudflare-tunnel.sh-10 +test-cloudflare-tunnel.sh-11' check_subset cloudflare 1 '' 'test-cloudflare-tunnel.sh' check_subset push-worker 7 '2' 'test-push-worker-registration.sh-2' check_subset push-worker 7 '0 1 2 3' 'test-push-worker-registration.sh-0 @@ -332,6 +352,8 @@ check_reject 'motd cut into four' motd 4 '' check_reject 'motd cut into eight' motd 8 '' check_reject 'motd cut into twelve' motd 12 '' check_reject 'cloudflare cut into four' cloudflare 4 '' +check_reject 'cloudflare cut into eight' cloudflare 8 '' +check_reject 'cloudflare cut into two' cloudflare 2 '' check_reject 'push-worker cut into eight' push-worker 8 '' check_reject 'push-worker cut into four' push-worker 4 '' check_reject 'push-worker cut into two' push-worker 2 '' @@ -367,6 +389,12 @@ done for bad in '0 1 2 3x' '4 5 6x' '7' '6 6' '+1' '010'; do check_bad_set push-worker 7 "$bad" done + +# The twelve-segment suite validates its own tokens the same way, so a set that +# only matches the old eight-segment layout cannot pass for a good one. +for bad in '0 1 2 3x' '0 2 6x' '12' '11 11' '+1' '010'; do + check_bad_set cloudflare 12 "$bad" +done for bad in 0 9 08 '4x' '1 4' 9223372036854775808 18446744073709551616; do : > "$CAPTURE" rc=0 From 038d6ee76fb619009fd92892493f509a4f23aebf Mon Sep 17 00:00:00 2001 From: FORGE Date: Sun, 25 Oct 2026 03:31:01 +0800 Subject: [PATCH 5/6] ci: fold cloudflare into two jobs to stay under the concurrency limit The twelve-segment split spread cloudflare over three jobs per OS, which put twenty-one jobs in the plan (nine per OS plus lint). GitHub Free runs twenty at a time, so cloudflare-a waited 44 s for a runner and the wall clock rose to 109 s. Cut the suite into two jobs instead, balancing the measured segment cost: - cloudflare-a: 2 3 5 6 11 -> 1+42+25+58+23 = 149 s - cloudflare-b: 0 1 4 7 8 9 10 -> 11+12+21+19+39+21+19 = 142 s Every segment still runs exactly once per OS, the heavy tail (42 s and 58 s) stays apart from the 39 s segment, and the plan drops to nineteen jobs, so no job queues behind another. tests/test-shards.sh expects the new pair; the segment count stays twelve, so the token and count rejection cases are unchanged. --- .github/workflows/shell-tests.yml | 28 ++++++++++------------------ tests/test-shards.sh | 18 +++++++++--------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index ae086d3..0d39284 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -70,9 +70,9 @@ jobs: include: # motd is cut into eleven segments spread over three jobs of four, # four and three; push-worker is cut into seven, two jobs of four and - # three; cloudflare is cut into twelve, three jobs of four whose - # segment sets mix early and late segments so the slowest one is - # never paired with the other heavy halves. + # three; cloudflare is cut into twelve, two jobs of five and seven + # balanced by measured cost, keeping the plan at nineteen jobs under + # the twenty-job concurrency instead of queueing a third job. - shard: motd-a suite: motd segments: 11 @@ -99,15 +99,11 @@ jobs: - shard: cloudflare-a suite: cloudflare segments: 12 - segment_set: "0 2 6 9" + segment_set: "2 3 5 6 11" - shard: cloudflare-b suite: cloudflare segments: 12 - segment_set: "1 3 5 7" - - shard: cloudflare-c - suite: cloudflare - segments: 12 - segment_set: "4 8 10 11" + segment_set: "0 1 4 7 8 9 10" - shard: other suite: other segments: 1 @@ -152,9 +148,9 @@ jobs: include: # motd is cut into eleven segments spread over three jobs of four, # four and three; push-worker is cut into seven, two jobs of four and - # three; cloudflare is cut into twelve, three jobs of four whose - # segment sets mix early and late segments so the slowest one is - # never paired with the other heavy halves. + # three; cloudflare is cut into twelve, two jobs of five and seven + # balanced by measured cost, keeping the plan at nineteen jobs under + # the twenty-job concurrency instead of queueing a third job. - shard: motd-a suite: motd segments: 11 @@ -181,15 +177,11 @@ jobs: - shard: cloudflare-a suite: cloudflare segments: 12 - segment_set: "0 2 6 9" + segment_set: "2 3 5 6 11" - shard: cloudflare-b suite: cloudflare segments: 12 - segment_set: "1 3 5 7" - - shard: cloudflare-c - suite: cloudflare - segments: 12 - segment_set: "4 8 10 11" + segment_set: "0 1 4 7 8 9 10" - shard: other suite: other segments: 1 diff --git a/tests/test-shards.sh b/tests/test-shards.sh index 95be616..d18a403 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -296,18 +296,18 @@ test-motd.sh-8 test-motd.sh-9 test-motd.sh-10' check_subset motd 1 '' 'test-motd.sh' -check_subset cloudflare 12 '0 2 6 9' 'test-cloudflare-tunnel.sh-0 -test-cloudflare-tunnel.sh-2 -test-cloudflare-tunnel.sh-6 -test-cloudflare-tunnel.sh-9' -check_subset cloudflare 12 '1 3 5 7' 'test-cloudflare-tunnel.sh-1 +check_subset cloudflare 12 '2 3 5 6 11' 'test-cloudflare-tunnel.sh-2 test-cloudflare-tunnel.sh-3 test-cloudflare-tunnel.sh-5 -test-cloudflare-tunnel.sh-7' -check_subset cloudflare 12 '4 8 10 11' 'test-cloudflare-tunnel.sh-4 -test-cloudflare-tunnel.sh-8 -test-cloudflare-tunnel.sh-10 +test-cloudflare-tunnel.sh-6 test-cloudflare-tunnel.sh-11' +check_subset cloudflare 12 '0 1 4 7 8 9 10' 'test-cloudflare-tunnel.sh-0 +test-cloudflare-tunnel.sh-1 +test-cloudflare-tunnel.sh-4 +test-cloudflare-tunnel.sh-7 +test-cloudflare-tunnel.sh-8 +test-cloudflare-tunnel.sh-9 +test-cloudflare-tunnel.sh-10' check_subset cloudflare 12 '' 'test-cloudflare-tunnel.sh-0 test-cloudflare-tunnel.sh-1 test-cloudflare-tunnel.sh-2 From d2bfa75327789a2567988d6afc95d6504f4aaa41 Mon Sep 17 00:00:00 2001 From: LUNO Date: Sun, 25 Oct 2026 03:37:16 +0800 Subject: [PATCH 6/6] docs: describe the current shard topology --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 76d9173..d96da57 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,12 @@ 生产宿主机(包括 Netcup)仅执行差异检查及改动脚本的 `bash -n`,不运行全量 ShellCheck 或完整测试集; 分析和测试交由 CI:PR 按完整差异及依赖映射选择套件,README/许可证修改无需业务测试; 公共设施或未知依赖改动回退全量,main 推送仍执行全量。两个必需 Job 始终运行,输出选择和跳过记录。 -每个 OS 的测试作业按七个 shard 并行:`motd-a`/`motd-b`(`test-motd.sh`,各跑四段)、 -`cloudflare-a`/`cloudflare-b`(`test-cloudflare-tunnel.sh`,各跑四段)、`push-worker` -(`test-push-worker-registration.sh`)、`push`(仅 `test-push.sh`)、`other`(其余全部套件); +每个 OS 的测试作业按九个 shard 并行:`motd-a`/`motd-b`/`motd-c`(`test-motd.sh`,11 段按 4/4/3 分配)、 +`cloudflare-a`/`cloudflare-b`(`test-cloudflare-tunnel.sh`,12 段按 5/7 分配)、`push-worker-a`/`push-worker-b` +(`test-push-worker-registration.sh`,7 段按 4/3 分配)、`push`(仅 `test-push.sh`)、`other`(其余全部套件); 本机全量用 `tests/run.sh all`。 重套件(motd、Cloudflare Tunnel、push worker 注册)按语义分段,import 守卫 `seg_is`:`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行(本机默认 1,即按原顺序串行)。 -motd 与 Cloudflare Tunnel 各切成八段:`-a` 作业用 `TEST_SEGMENT_SET="0 1 2 3"`、`-b` 作业用 `"4 5 6 7"`,两个作业的并集恰好覆盖全部段且互不重叠;push worker 注册仍为四段、单作业。 +motd 切成十一段、Cloudflare Tunnel 切成十二段、push worker 注册切成七段:每个作业用 `TEST_SEGMENT_SET` 只跑自己的段,同一套件的所有作业并集恰好覆盖全部段且互不重叠;任一作业的段数、下标或重复项不合法时,在任何套件启动前以状态 2 退出。 `TEST_SEGMENT_SET` 只接受互不重复、且在 `TEST_SEGMENTS` 范围内的段下标,非法值时在任何套件启动前以状态 2 退出;未设置则照旧跑全部段。 Ubuntu 使用固定 ShellCheck v0.10.0:所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛,tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛;新目录中的生产 .sh 文件自动纳入,不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败,Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由;禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable,禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退;warning 门槛失败时后续 Ubuntu 行为套件不会执行,不能以 Debian 通过代替双环境验收。