diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index be5a514..0d39284 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -45,10 +45,19 @@ jobs: shell: bash run: | # All tracked Bash scripts outside tests are production by default. - # Check both groups even when production lint fails. + # Both groups lint in parallel, one file per process, and both are + # started before the first wait so a failure in one cannot hide the + # other. pipefail stays on, so a failed file enumeration fails its + # group even though xargs then sees empty input; any failure becomes + # a single exit 1. + set -o pipefail status=0 - git ls-files -z -- "*.sh" ":!:tests/**" | xargs -0 -r shellcheck -x --severity=warning -- || status=1 - git ls-files -z -- "tests/*.sh" | xargs -0 -r shellcheck -x --severity=error -- || status=1 + git ls-files -z -- "*.sh" ":!:tests/**" | xargs -0 -r -n1 -P4 shellcheck -x --severity=warning -- & + production_pid=$! + git ls-files -z -- "tests/*.sh" | xargs -0 -r -n1 -P4 shellcheck -x --severity=error -- & + test_pid=$! + wait "$production_pid" || status=1 + wait "$test_pid" || status=1 exit "$status" ubuntu-tests: @@ -59,30 +68,42 @@ jobs: fail-fast: false matrix: include: - # motd and cloudflare are cut into eight segments; each of their two - # jobs runs four of them, so a single suite spans two jobs. + # motd is cut into eleven segments spread over three jobs of four, + # four and three; push-worker is cut into seven, two jobs of four and + # three; cloudflare is cut into twelve, two jobs of five and seven + # balanced by measured cost, keeping the plan at nineteen jobs under + # the twenty-job concurrency instead of queueing a third job. - shard: motd-a suite: motd - segments: 8 + segments: 11 segment_set: "0 1 2 3" - shard: motd-b suite: motd - segments: 8 + segments: 11 segment_set: "4 5 6 7" - - shard: push-worker + - shard: motd-c + suite: motd + segments: 11 + segment_set: "8 9 10" + - shard: push-worker-a + suite: push-worker + segments: 7 + segment_set: "0 1 2 3" + - shard: push-worker-b suite: push-worker - segments: 4 + segments: 7 + segment_set: "4 5 6" - shard: push suite: push segments: 1 - shard: cloudflare-a suite: cloudflare - segments: 8 - segment_set: "0 1 2 3" + segments: 12 + segment_set: "2 3 5 6 11" - shard: cloudflare-b suite: cloudflare - segments: 8 - segment_set: "4 5 6 7" + segments: 12 + segment_set: "0 1 4 7 8 9 10" - shard: other suite: other segments: 1 @@ -125,30 +146,42 @@ jobs: fail-fast: false matrix: include: - # motd and cloudflare are cut into eight segments; each of their two - # jobs runs four of them, so a single suite spans two jobs. + # motd is cut into eleven segments spread over three jobs of four, + # four and three; push-worker is cut into seven, two jobs of four and + # three; cloudflare is cut into twelve, two jobs of five and seven + # balanced by measured cost, keeping the plan at nineteen jobs under + # the twenty-job concurrency instead of queueing a third job. - shard: motd-a suite: motd - segments: 8 + segments: 11 segment_set: "0 1 2 3" - shard: motd-b suite: motd - segments: 8 + segments: 11 segment_set: "4 5 6 7" - - shard: push-worker + - shard: motd-c + suite: motd + segments: 11 + segment_set: "8 9 10" + - shard: push-worker-a + suite: push-worker + segments: 7 + segment_set: "0 1 2 3" + - shard: push-worker-b suite: push-worker - segments: 4 + segments: 7 + segment_set: "4 5 6" - shard: push suite: push segments: 1 - shard: cloudflare-a suite: cloudflare - segments: 8 - segment_set: "0 1 2 3" + segments: 12 + segment_set: "2 3 5 6 11" - shard: cloudflare-b suite: cloudflare - segments: 8 - segment_set: "4 5 6 7" + segments: 12 + segment_set: "0 1 4 7 8 9 10" - shard: other suite: other segments: 1 diff --git a/README.md b/README.md index 76d9173..d96da57 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,12 @@ 生产宿主机(包括 Netcup)仅执行差异检查及改动脚本的 `bash -n`,不运行全量 ShellCheck 或完整测试集; 分析和测试交由 CI:PR 按完整差异及依赖映射选择套件,README/许可证修改无需业务测试; 公共设施或未知依赖改动回退全量,main 推送仍执行全量。两个必需 Job 始终运行,输出选择和跳过记录。 -每个 OS 的测试作业按七个 shard 并行:`motd-a`/`motd-b`(`test-motd.sh`,各跑四段)、 -`cloudflare-a`/`cloudflare-b`(`test-cloudflare-tunnel.sh`,各跑四段)、`push-worker` -(`test-push-worker-registration.sh`)、`push`(仅 `test-push.sh`)、`other`(其余全部套件); +每个 OS 的测试作业按九个 shard 并行:`motd-a`/`motd-b`/`motd-c`(`test-motd.sh`,11 段按 4/4/3 分配)、 +`cloudflare-a`/`cloudflare-b`(`test-cloudflare-tunnel.sh`,12 段按 5/7 分配)、`push-worker-a`/`push-worker-b` +(`test-push-worker-registration.sh`,7 段按 4/3 分配)、`push`(仅 `test-push.sh`)、`other`(其余全部套件); 本机全量用 `tests/run.sh all`。 重套件(motd、Cloudflare Tunnel、push worker 注册)按语义分段,import 守卫 `seg_is`:`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行(本机默认 1,即按原顺序串行)。 -motd 与 Cloudflare Tunnel 各切成八段:`-a` 作业用 `TEST_SEGMENT_SET="0 1 2 3"`、`-b` 作业用 `"4 5 6 7"`,两个作业的并集恰好覆盖全部段且互不重叠;push worker 注册仍为四段、单作业。 +motd 切成十一段、Cloudflare Tunnel 切成十二段、push worker 注册切成七段:每个作业用 `TEST_SEGMENT_SET` 只跑自己的段,同一套件的所有作业并集恰好覆盖全部段且互不重叠;任一作业的段数、下标或重复项不合法时,在任何套件启动前以状态 2 退出。 `TEST_SEGMENT_SET` 只接受互不重复、且在 `TEST_SEGMENTS` 范围内的段下标,非法值时在任何套件启动前以状态 2 退出;未设置则照旧跑全部段。 Ubuntu 使用固定 ShellCheck v0.10.0:所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛,tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛;新目录中的生产 .sh 文件自动纳入,不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败,Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由;禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable,禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退;warning 门槛失败时后续 Ubuntu 行为套件不会执行,不能以 Debian 通过代替双环境验收。 diff --git a/tests/run.sh b/tests/run.sh index b787d72..a15213c 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -71,14 +71,14 @@ done # defines. TEST_SEGMENTS sets how many processes share the work; with # TEST_SEGMENTS=1 every suite runs in one process, all segments in order. declare -rA SEGMENTED_SUITES=( - [test-motd.sh]=8 - [test-cloudflare-tunnel.sh]=8 - [test-push-worker-registration.sh]=4 + [test-motd.sh]=11 + [test-cloudflare-tunnel.sh]=12 + [test-push-worker-registration.sh]=7 ) segments=${TEST_SEGMENTS:-1} -# One digit, at most the eight segments a suite actually has: no accepted value -# can overflow shell arithmetic and every index below stays a single character. -[[ "$segments" =~ ^[1-8]$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } +# Bounded lexical form first, so no accepted value can overflow shell arithmetic +# or smuggle in a sign, a prefix or a multi-digit index past the range checks. +[[ "$segments" =~ ^([1-9]|1[0-2])$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } # TEST_SEGMENT_SET narrows one job to a subset of the segments ("0 1 2 3"), so a # single suite can be spread over several jobs. Unset or blank keeps every @@ -94,7 +94,7 @@ if [[ -n "${TEST_SEGMENT_SET:-}" && -n "${TEST_SEGMENT_SET//[[:space:]]/}" ]]; t segment_value=${segment_value//$'\n'/ } read -r -a segment_entries <<< "$segment_value" for segment_index in "${segment_entries[@]}"; do - [[ "$segment_index" =~ ^[0-7]$ ]] \ + [[ "$segment_index" =~ ^([0-9]|1[01])$ ]] \ || { printf 'Invalid TEST_SEGMENT_SET entry: %s\n' "$segment_index" >&2; exit 2; } (( segment_index < segments )) \ || { printf 'Segment %s out of range for TEST_SEGMENTS=%s\n' "$segment_index" "$segments" >&2; exit 2; } diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh index 2e48726..a11b519 100644 --- a/tests/test-cloudflare-tunnel.sh +++ b/tests/test-cloudflare-tunnel.sh @@ -507,8 +507,11 @@ find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -qu pass "configure then real repository uninstall removes source and preserves keyring" fi -if seg_is 3; then -for signal in HUP INT TERM; do +# The uninstall signal matrix is 39 independent cases (3 signals x 13 phases), +# each building its own fixture. Segments 3 and 4 split it by signal, so the +# serial order stays HUP, INT, TERM exactly as before. +uninstall_signal_phase_cases() { + local signal=$1 phase case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do new_case @@ -537,10 +540,18 @@ for signal in HUP INT TERM; do release_repository_lock || fail "uninstall $signal at $phase lock cleanup failed" pass "uninstall $signal=$expected cleans $phase" done -done +} + +if seg_is 3; then +uninstall_signal_phase_cases HUP +uninstall_signal_phase_cases INT fi if seg_is 4; then +uninstall_signal_phase_cases TERM +fi + +if seg_is 5; then new_case set_old_generation configure_repository @@ -667,7 +678,7 @@ for schema_case in version generation duplicate unknown order state mode uid gid done fi -if seg_is 5; then +if seg_is 6; then for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do ( new_case @@ -966,7 +977,7 @@ for restore_id in auto_update_script auto_update_service auto_update_timer sourc done fi -if seg_is 6; then +if seg_is 7; then for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do for bad_metadata in mode owner gid symlink directory fifo; do ( @@ -998,6 +1009,9 @@ for target_id in auto_update_script auto_update_service auto_update_timer source pass "capture rejects $target_id $bad_metadata without deleting target" done done +fi + +if seg_is 8; then ( unset CLOUDFLARED_TRUST_ANCHOR @@ -1049,7 +1063,7 @@ for creation_phase in before-create-marker before-create-manifest-stage before-c done fi -if seg_is 7; then +if seg_is 9; then for bookkeeping_fault in marker stage rename; do ( new_case @@ -1148,6 +1162,9 @@ for capture_file in old-key old-source old-current; do pass "repository capture $capture_file $capture_fault preserves formal bytes and metadata" done done +fi + +if seg_is 10; then ( new_case @@ -1256,6 +1273,10 @@ for restore_object in key source current; do done done +fi + +if seg_is 11; then + for operation in install_cloudflared upgrade_cloudflared; do ( new_case diff --git a/tests/test-lint-coverage.sh b/tests/test-lint-coverage.sh index bb14d00..e210652 100644 --- a/tests/test-lint-coverage.sh +++ b/tests/test-lint-coverage.sh @@ -16,14 +16,20 @@ awk ' block { step=0; block=0 } END { if (found != 1 || lines == 0) exit 1 } ' "$ROOT_DIR/.github/workflows/shell-tests.yml" > "$root/lint.sh" +# Both groups must keep the per-file, four-way parallel invocation. +if [[ "$(grep -c -- '-n1 -P4' "$root/lint.sh")" != 2 ]]; then + printf 'the lint block no longer starts both groups with -n1 -P4\n' >&2 + exit 1 +fi mkdir -p "$root/bin" "$root/repo" cat > "$root/bin/shellcheck" <<'STUB' #!/usr/bin/env bash set -euo pipefail -[[ $1 == -x && $3 == -- && $# -gt 3 ]] || exit 90 +# One file per call (xargs -n1), classified by severity: anything else is a defect. +[[ $# -eq 4 && $1 == -x && $3 == -- ]] || exit 90 level=$2 shift 3 -for path; do printf "%s\t%s\n" "$level" "$path" >> "$CAPTURE"; done +printf "%s\t%s\n" "$level" "$1" >> "$CAPTURE" case $level in --severity=warning) exit "$PRODUCTION_STATUS" ;; --severity=error) exit "$TEST_STATUS" ;; @@ -31,6 +37,20 @@ case $level in esac STUB chmod +x "$root/bin/shellcheck" +real_git=$(command -v git) +cat > "$root/bin/git" < "$root/production.expected" +printf "%s\n" \ $'--severity=error\ttests/direct.sh' \ - $'--severity=error\ttests/nested/deep.sh' > "$root/expected" + $'--severity=error\ttests/nested/deep.sh' > "$root/tests.expected" +cat "$root/production.expected" "$root/tests.expected" > "$root/all.expected" +# Multiset comparison: every tracked script of both groups exactly once, with +# its own group's severity. Sorting keeps the parallel order out of the verdict, +# while a duplicated or a missing file still shows up as a difference. +check_linted() { + local label=$1 expected=$2 + LC_ALL=C sort "$root/capture" > "$root/capture.sorted" + LC_ALL=C sort "$expected" > "$root/expected.sorted" + if ! diff -u "$root/expected.sorted" "$root/capture.sorted"; then + printf 'FAIL: %s: linted file set is not the tracked split\n' "$label" >&2 + exit 1 + fi +} +run_lint() { + local status=0 + bash --noprofile --norc -e -o pipefail "$root/lint.sh" || status=$? + printf '%s' "$status" +} +# Both groups must always be linted in full, and a failure in either one must +# fail the step: the four result combinations cover that. +export ENUM_FAIL=none for PRODUCTION_STATUS in 0 1; do for TEST_STATUS in 0 1; do export PRODUCTION_STATUS TEST_STATUS : > "$CAPTURE" - status=0 - bash --noprofile --norc -e -o pipefail "$root/lint.sh" || status=$? + status=$(run_lint) expected=0 if (( PRODUCTION_STATUS != 0 || TEST_STATUS != 0 )); then expected=1; fi - [[ $status == "$expected" ]] - diff -u "$root/expected" "$CAPTURE" - printf "PASS: workflow lint production=%s tests=%s exit=%s\n" "$PRODUCTION_STATUS" "$TEST_STATUS" "$status" + [[ $status == "$expected" ]] \ + || { printf 'FAIL: lint exited %s, expected %s\n' "$status" "$expected" >&2; exit 1; } + check_linted "production=$PRODUCTION_STATUS tests=$TEST_STATUS" "$root/all.expected" + printf "PASS: workflow lint production=%s tests=%s exit=%s\n" \ + "$PRODUCTION_STATUS" "$TEST_STATUS" "$status" done done +# A failed enumeration must fail the step without hiding the other group: the +# background pipeline keeps pipefail, so git failing is not swallowed by the +# empty xargs run behind it. +for ENUM_FAIL in production tests both; do + export ENUM_FAIL PRODUCTION_STATUS=0 TEST_STATUS=0 + : > "$CAPTURE" + status=$(run_lint) + [[ $status == 1 ]] \ + || { printf 'FAIL: %s enumeration failure exited %s\n' "$ENUM_FAIL" "$status" >&2; exit 1; } + case "$ENUM_FAIL" in + production) check_linted "$ENUM_FAIL enumeration failure" "$root/tests.expected" ;; + tests) check_linted "$ENUM_FAIL enumeration failure" "$root/production.expected" ;; + both) [[ ! -s "$CAPTURE" ]] \ + || { printf 'FAIL: both enumerations failed but files were linted\n' >&2; exit 1; } ;; + esac + printf 'PASS: %s enumeration failure exits 1\n' "$ENUM_FAIL" +done diff --git a/tests/test-motd.sh b/tests/test-motd.sh index a7ea717..1001493 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -191,6 +191,68 @@ motd_snapshot_capture_hook() { motd_run_locked_operation install CHILD chmod 0700 "$TEST_DIR/building-signal-child.sh" +run_rollback_damage_cases() { + local rollback_impl=$1 rollback_damage root + for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do + ( + root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" + if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install + case "$rollback_damage" in + motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; + absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + before=$(target_hash "$root") + before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + motd_rollback_hook() { + local id="$1" path target + [[ "$id" == motd_news ]] || return 0 + case "$rollback_damage" in + motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) + path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" + cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" + ;; + absent-regular|absent-symlink|absent-fifo) + ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + rm() { + local status + command rm "$@"; status=$?; unset -f rm + case "$rollback_damage" in + absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; + absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; + absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; + esac + return "$status" + } + ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + return 0 + } + assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" + transaction=$(cat "$MOTD_STATE_DIR/pending") + journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" + grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" + [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" + pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" + motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } + motd_take_lock; motd_reconcile_state; motd_release_lock + assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" + assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" + motd_take_lock; motd_release_lock + ) + done +} + if seg_is 0; then extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; @@ -934,8 +996,9 @@ for pending_hook_case in id extra-line inode; do assert_no_motd_residue "$root" ) done +fi - +if seg_is 5; then for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do ( root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" @@ -969,7 +1032,7 @@ for building_phase in transaction-snapshot-building capture-motd capture-issue_n done fi -if seg_is 5; then +if seg_is 6; then for previous_impl in tool module; do ( root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" @@ -1065,7 +1128,7 @@ done ) fi -if seg_is 6; then +if seg_is 7; then for missing_command in bash chown flock ln od readlink sha256sum touch wc; do ( root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" @@ -1138,7 +1201,9 @@ for group_impl in tool module; do ) done done +fi +if seg_is 8; then for illegal_legacy_name in motd.previous-unknown issue.previous-unknown motd.invalid-backup evil.initial-backup; do ( root="$TEST_DIR/legacy-name-${illegal_legacy_name//[^a-zA-Z0-9]/-}"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install @@ -1200,68 +1265,12 @@ done ) fi -if seg_is 7; then -for rollback_impl in tool module; do - for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do - ( - root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" - if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install - case "$rollback_damage" in - motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; - absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - before=$(target_hash "$root") - before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - motd_rollback_hook() { - local id="$1" path target - [[ "$id" == motd_news ]] || return 0 - case "$rollback_damage" in - motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) - path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" - cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" - ;; - absent-regular|absent-symlink|absent-fifo) - ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - rm() { - local status - command rm "$@"; status=$?; unset -f rm - case "$rollback_damage" in - absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; - absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; - absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; - esac - return "$status" - } - ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - return 0 - } - assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" - transaction=$(cat "$MOTD_STATE_DIR/pending") - journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" - grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" - [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" - pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" - motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } - motd_take_lock; motd_reconcile_state; motd_release_lock - assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" - assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" - motd_take_lock; motd_release_lock - ) - done +if seg_is 9; then +run_rollback_damage_cases tool +fi -done +if seg_is 10; then +run_rollback_damage_cases module for rollback_impl in tool module; do ( diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh index acb194e..055ffc3 100755 --- a/tests/test-push-worker-registration.sh +++ b/tests/test-push-worker-registration.sh @@ -214,6 +214,187 @@ EOF chmod 0700 "$root/bin"/* } +# Two case bodies below are long enough to be shared by a pair of segments: the +# retry stress and the failed-reap runs. Each body lives here once and both +# segments run a contiguous half of the runs, so no case body is duplicated and +# no PASS summary is repeated. Only the half holding the last run prints the +# total, which keeps the summary text identical to the unsegmented suite. +run_preserved_cleanup_retry_case() ( + local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file + local owner watchdog rc=0 metadata="" state_status=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + credential="$runtime/private-key" + printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" + RUNTIME_KEY_FILE=$credential + + setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do + read_process_record "$leader" || true + [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break + sleep 0.01 + done + read_process_record "$leader" + leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start + WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid + publish_worker_session_state active + state_file=$WORKER_SESSION_STATE_FILE + + eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" + terminate_managed_session() { return 1; } + owner=$BASHPID + test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! + cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + + (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" + [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" + [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || + fail "retry run $run lost runtime identity" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + metadata=$(stat -Lc '%u:%g:%a' -- "$credential") + [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" + grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" + state_status=0; read_worker_session_state "$state_file" || state_status=$? + assert_eq 0 "$state_status" "retry run $run retained state parses" + [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && + "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || + fail "retry run $run retained state identity mismatch" + [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && + "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" + test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" + read_process_record "$leader" + [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" + + eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" + cleanup_runtime + wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" + [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || + fail "retry run $run left state or runtime" + runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" + [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" + pass "failed cleanup evidence retry run $run preserves then clears runtime" +) + +runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} +[[ "$runtime_evidence_stress" =~ ^[0-9]+$ ]] \ + || fail "invalid PUSH_RUNTIME_EVIDENCE_STRESS: $runtime_evidence_stress" +# The ceiling keeps an odd count working: one half always holds the last run. +runtime_evidence_split=$(( (10#$runtime_evidence_stress + 1) / 2 )) + +# The failed-reap child and case body are shared for the same reason: part one +# of segment 0 costs about 18 s while each seeded run costs about 2.8 s, so the +# twenty runs split 8/12 and both segments stay inside the time budget. +cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" mode="$3" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime +setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! +for _ in {1..100}; do + leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) + leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) + leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) + [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break + sleep 0.01 +done +[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 +printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" +WORKER_TRANSFER_PID=$leader +WORKER_TRANSFER_START=$leader_start +WORKER_TRANSFER_PGID=$leader_pgid +WORKER_TRANSFER_SID=$leader_sid +publish_worker_session_state active +state_file=$WORKER_SESSION_STATE_FILE +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { return 1; } +start_ns=$(date +%s%N) +rc=0 +terminate_worker_transfer || rc=$? +elapsed_ns=$(( $(date +%s%N) - start_ns )) +[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 +[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 +read_worker_session_state "$state_file" +[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 +next_rc=0 +run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? +[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 +printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" +: > "$root/capture/returned" +if [[ "$mode" == hold ]]; then + while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done + for _ in {1..400}; do + reap_status=0 + reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? + case "$reap_status" in + 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; + *) sleep 0.01 ;; + esac + done + (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 + : > "$root/capture/reaped" + for _ in {1..400}; do + [[ ! -e "$state_file" && ! -L "$state_file" ]] && break + sleep 0.01 + done + [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 + terminate_managed_session() { original_terminate_managed_session "$@"; } +else + for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done + [[ -e "$root/capture/release-recovery" ]] || exit 97 + terminate_managed_session() { original_terminate_managed_session "$@"; } +fi +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/failed-reap-child.sh" + +run_failed_reap_case() { + local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 + bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! + test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! + for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" + : > "$root/capture/release-recovery" + wait "$child" || rc=$? + [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true + assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" + [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" +} + +# The state-publication matrix is shared by segments 5 and 6 the same way: the +# combinations keep their original state, phase and signal order and split into +# two contiguous halves, so overriding the variables below still puts every +# combination in exactly one half. +publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} +publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} +publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} +publication_total=0 +for publication_state in $publication_states; do + for publication_phase in $publication_phases; do + for publication_signal in $publication_signals; do + publication_total=$(( publication_total + 1 )) + done + done +done +publication_split=$(( (publication_total + 1) / 2 )) + if seg_is 0; then source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' @@ -576,89 +757,23 @@ done pass "managed leader PID reuse is treated as original session gone" ) -cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" mode="$3" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime -setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! -for _ in {1..100}; do - leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) - leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) - leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) - [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break - sleep 0.01 -done -[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 -printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" -WORKER_TRANSFER_PID=$leader -WORKER_TRANSFER_START=$leader_start -WORKER_TRANSFER_PGID=$leader_pgid -WORKER_TRANSFER_SID=$leader_sid -publish_worker_session_state active -state_file=$WORKER_SESSION_STATE_FILE -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { return 1; } -start_ns=$(date +%s%N) -rc=0 -terminate_worker_transfer || rc=$? -elapsed_ns=$(( $(date +%s%N) - start_ns )) -[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 -[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 -read_worker_session_state "$state_file" -[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 -next_rc=0 -run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? -[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 -printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" -: > "$root/capture/returned" -if [[ "$mode" == hold ]]; then - while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done - for _ in {1..400}; do - reap_status=0 - reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? - case "$reap_status" in - 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; - *) sleep 0.01 ;; - esac - done - (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 - : > "$root/capture/reaped" - for _ in {1..400}; do - [[ ! -e "$state_file" && ! -L "$state_file" ]] && break - sleep 0.01 - done - [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 - terminate_managed_session() { original_terminate_managed_session "$@"; } -else - for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done - [[ -e "$root/capture/release-recovery" ]] || exit 97 - terminate_managed_session() { original_terminate_managed_session "$@"; } +# First half of the twenty failed-reap runs. Segment 1 starts at run 9 and +# holds the summary, so the runs keep their original order and the summary +# still appears exactly once. +for failed_reap_run in $(seq 1 8); do run_failed_reap_case "$failed_reap_run"; done fi -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/failed-reap-child.sh" -run_failed_reap_case() { - local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 - bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! - test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! - for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" - : > "$root/capture/release-recovery" - wait "$child" || rc=$? - [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true - assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" - [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" -} -for failed_reap_run in $(seq 1 20); do run_failed_reap_case "$failed_reap_run"; done +if seg_is 1; then +# PUSH_REAP_ONLY keeps its original meaning for the whole suite: with it set, +# nothing after the deadline case runs. That used to be one exit inside +# segment 0, so the second half carries the same guard. +if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then + exit 0 +fi + +# Second half of the twenty failed-reap runs, then the parent fallback that +# follows them in the original order. +for failed_reap_run in $(seq 9 20); do run_failed_reap_case "$failed_reap_run"; done pass "failed managed-session reap remains bounded for 20 deterministic runs" ( @@ -692,7 +807,7 @@ pass "failed managed-session reap remains bounded for 20 deterministic runs" ) fi -if seg_is 1; then +if seg_is 2; then ( trap - EXIT HUP INT TERM sleep 60 & replacement=$! @@ -780,81 +895,27 @@ chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" pass "hard timeout cleanup failure is bounded and blocks subsequent command" ) -run_preserved_cleanup_retry_case() ( - local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file - local owner watchdog rc=0 metadata="" state_status=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - credential="$runtime/private-key" - printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" - RUNTIME_KEY_FILE=$credential - - setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do - read_process_record "$leader" || true - [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break - sleep 0.01 - done - read_process_record "$leader" - leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start - WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid - publish_worker_session_state active - state_file=$WORKER_SESSION_STATE_FILE - - eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" - terminate_managed_session() { return 1; } - owner=$BASHPID - test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! - cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - - (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" - [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" - [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || - fail "retry run $run lost runtime identity" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - metadata=$(stat -Lc '%u:%g:%a' -- "$credential") - [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" - grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" - state_status=0; read_worker_session_state "$state_file" || state_status=$? - assert_eq 0 "$state_status" "retry run $run retained state parses" - [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && - "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || - fail "retry run $run retained state identity mismatch" - [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && - "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" - test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" - read_process_record "$leader" - [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" - - eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" - cleanup_runtime - wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" - [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || - fail "retry run $run left state or runtime" - runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" - [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" - pass "failed cleanup evidence retry run $run preserves then clears runtime" -) +# First half of the shared retry stress; the summary stays with the last run, +# so a lower PUSH_RUNTIME_EVIDENCE_STRESS still reports its total once. +for runtime_evidence_run in $(seq 1 "$runtime_evidence_split"); do + run_preserved_cleanup_retry_case "$runtime_evidence_run" +done +if (( runtime_evidence_split >= 10#$runtime_evidence_stress )); then + pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +fi +fi -runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} -for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do +if seg_is 3; then +# Second half of the shared retry stress. +for runtime_evidence_run in $(seq $(( runtime_evidence_split + 1 )) "$runtime_evidence_stress"); do run_preserved_cleanup_retry_case "$runtime_evidence_run" done -pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +if (( runtime_evidence_split < 10#$runtime_evidence_stress )); then + pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +fi fi -if seg_is 2; then +if seg_is 4; then ( local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE @@ -1338,7 +1399,7 @@ run_active_grace_signal_case INT 130 run_active_grace_signal_case TERM 143 fi -if seg_is 3; then +if seg_is 5; then cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1413,6 +1474,8 @@ run_live_cleanup_failure_signal_case HUP 129 run_live_cleanup_failure_signal_case INT 130 run_live_cleanup_failure_signal_case TERM 143 +# Both publication halves carry their own child fixture and case runner: the +# two halves are separate processes, so neither may rely on the other. cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1510,12 +1573,130 @@ run_state_publication_signal_case() ( trap - EXIT pass "$state/$phase/$signal_name publication window cleans process tree and runtime" ) -publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} -publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} -publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} + +# First half of the shared publication matrix, in the original state, phase and +# signal order. +publication_index=0 for publication_state in $publication_states; do for publication_phase in $publication_phases; do for publication_signal in $publication_signals; do + publication_index=$(( publication_index + 1 )) + (( publication_index <= publication_split )) || break 3 + case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac + run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" + done + done +done +fi + +if seg_is 6; then +# Both publication halves carry their own child fixture and case runner: the +# two halves are separate processes, so neither may rely on the other. +cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +main_owner=$BASHPID +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$BASHPID" != "$main_owner" ]]; then return 1; fi + original_terminate_managed_session "$@" +} +worker_session_state_publish_hook() { + local phase="$1" state="$2" state_file="$3" hook_stage="$4" marker_stage worker_start + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$state" == active && "$phase" == state-file-assigned ]]; then + : > "$STATE_CLEANUP_RELEASE" + fi + [[ "$phase" == "$STATE_PUBLISH_PHASE" && "$state" == "$STATE_PUBLISH_STATE" ]] || return 0 + [[ ! -e "$STATE_PUBLISH_MARKER" ]] || return 0 + worker_start="" + for _ in {1..20}; do + if read_process_record "$BASHPID" && [[ "$PROC_UID" == "$EUID" && "$PROC_STATE" != Z ]]; then worker_start=$PROC_START; break; fi + sleep 0.01 + done + [[ -n "$worker_start" ]] || return 1 + marker_stage="$STATE_PUBLISH_MARKER.$BASHPID.stage" + (umask 077; set -o noclobber; : > "$marker_stage") || return 1 + printf 'phase=%s\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\nstate_file=%s\nhook_stage=%s\n' \ + "$phase" "$state" "$BASHPID" "$worker_start" "$WORKER_TRANSFER_PID" "$WORKER_TRANSFER_START" \ + "$WORKER_TRANSFER_PGID" "$WORKER_TRANSFER_SID" "$state_file" "$hook_stage" > "$marker_stage" + chmod 0600 "$marker_stage"; mv -fT "$marker_stage" "$STATE_PUBLISH_MARKER" + while :; do :; done +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/state-publication-child.sh" + +run_state_publication_signal_case() ( + local state="$1" phase="$2" signal_name="$3" expected="$4" mode root main_pid="" main_start="" worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" + local watchdog="" watchdog_start="" unrelated="" unrelated_start="" rc=0 line key value marker state_file hook_stage + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + cleanup_publication_fixture() { + [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } + [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } + [[ -z "$watchdog" || -z "$watchdog_start" ]] || { test_process_identity_exists "$watchdog" "$watchdog_start" && kill -TERM "$watchdog" 2>/dev/null || true; } + [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } + [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true + [[ -z "$watchdog" ]] || wait "$watchdog" 2>/dev/null || true + [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true + } + trap cleanup_publication_fixture EXIT + mode=active; [[ "$state" == cleanup_failed ]] && mode=cleanup + root="$TEST_DIR/state-publish-$state-$phase-$signal_name${5:-}"; CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + mkdir -m 0700 "$root/capture" + marker="$root/capture/publish-marker" + sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + STATE_PUBLISH_MODE="$mode" STATE_PUBLISH_STATE="$state" STATE_PUBLISH_PHASE="$phase" STATE_PUBLISH_MARKER="$marker" \ + STATE_CLEANUP_RELEASE="$root/capture/cleanup-release" bash "$TEST_DIR/state-publication-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$!; main_start=$(wait_test_process_start "$main_pid") + for _ in {1..600}; do [[ -f "$marker" ]] && break; sleep 0.05; done + [[ -f "$marker" && ! -L "$marker" && "$(stat -Lc %a "$marker")" == 600 ]] || { cat "$root/output.log" >&2 || true; fail "$state/$phase/$signal_name did not reach publish hook"; } + while IFS= read -r line; do key=${line%%=*}; value=${line#*=}; case "$key" in worker_pid) worker_pid=$value;;worker_start) worker_start=$value;;leader_pid) leader_pid=$value;;leader_start) leader_start=$value;;sid) managed_sid=$value;;state_file) state_file=$value;;hook_stage) hook_stage=$value;;esac; done < "$marker" + [[ "$worker_pid" =~ ^[1-9][0-9]*$ && "$worker_start" =~ ^[1-9][0-9]*$ && "$managed_sid" =~ ^[1-9][0-9]*$ ]] || fail "$state/$phase/$signal_name marker identity malformed" + wait_test_process_identity_present "$worker_pid" "$worker_start" || { cat "$marker" >&2; fail "$state/$phase/$signal_name worker identity missing at hook"; } + test_watchdog_process 30 "$root/watchdog-timeout" "$main_pid" & watchdog=$!; watchdog_start=$(wait_test_process_start "$watchdog") + printf "DIAG: publication state=%s phase=%s signal=%s main=%s/%s worker=%s/%s sid=%s\n" \ + "$state" "$phase" "$signal_name" "$main_pid" "$main_start" "$worker_pid" "$worker_start" "$managed_sid" + kill "-$signal_name" "$main_pid"; wait "$main_pid" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-timeout" ]] || fail "$state/$phase/$signal_name watchdog fired" + assert_eq "$expected" "$rc" "$state/$phase preserves $signal_name status" + wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$state/$phase/$signal_name left worker identity" + if [[ "$leader_pid" =~ ^[1-9][0-9]*$ && "$leader_start" =~ ^[1-9][0-9]*$ ]]; then + if ! wait_test_process_identity_gone "$leader_pid" "$leader_start"; then + read_process_record "$leader_pid" || true + printf 'leader remains pid=%s expected_start=%s actual_start=%s state=%s ppid=%s pgid=%s sid=%s\n' \ + "$leader_pid" "$leader_start" "${PROC_START:-unknown}" "${PROC_STATE:-unknown}" "${PROC_PPID:-unknown}" "${PROC_PGID:-unknown}" "${PROC_SID:-unknown}" >&2 + cat "/proc/$leader_pid/stat" >&2 2>/dev/null || true + fail "$state/$phase/$signal_name left leader identity" + fi + fi + collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$state/$phase/$signal_name cannot verify managed SID" + (( ${#session_pids[@]} == 0 )) || fail "$state/$phase/$signal_name left managed SID member: ${session_pids[*]}" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$state/$phase/$signal_name left runtime/session/registration state" + [[ ! -e "$state_file" && ! -e "$hook_stage" && ! -L "$state_file" && ! -L "$hook_stage" ]] || fail "$state/$phase/$signal_name left publication path" + test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$state/$phase/$signal_name killed unrelated process" + kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true + unrelated=""; unrelated_start=""; main_pid=""; main_start=""; watchdog=""; watchdog_start="" + trap - EXIT + pass "$state/$phase/$signal_name publication window cleans process tree and runtime" +) + +# Second half of the shared publication matrix. +publication_index=0 +for publication_state in $publication_states; do + for publication_phase in $publication_phases; do + for publication_signal in $publication_signals; do + publication_index=$(( publication_index + 1 )) + (( publication_index > publication_split )) || continue case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" done @@ -1568,4 +1749,3 @@ done printf 'All push worker registration tests passed.\n' fi - diff --git a/tests/test-shards.sh b/tests/test-shards.sh index c8bde8d..d18a403 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -142,7 +142,8 @@ jobs=$(awk ' # covered exactly once, union 0..N-1. check_matrix() { local job=$1 rows suite row_suite suite_segments segment_set_value - local declared covered index count total expected actual + local declared covered index count total expected actual token + local -a set_tokens=() rows=$(printf '%s\n' "$jobs" | awk -F'|' -v job="$job" '$1 == job') [[ -n "$rows" ]] || fail "no matrix rows for $job" if printf '%s\n' "$rows" | cut -d'|' -f2 | sort | uniq -d | grep -q .; then @@ -156,6 +157,18 @@ check_matrix() { case "$suite_segments" in ''|*[!0-9]*) fail "$job $suite has an unsupported segment count: '$suite_segments'" ;; esac + if [[ -n "$segment_set_value" ]]; then + # Strict tokens, checked before any coverage arithmetic: a set + # carries decimal indices of the declared range, so a trailing + # "x" (or "10x") fails here instead of being filtered out. + read -r -a set_tokens <<< "$segment_set_value" + for token in "${set_tokens[@]}"; do + [[ "$token" =~ ^([0-9]|1[01])$ ]] \ + || fail "$job $suite has an invalid segment_set entry: '$token'" + (( token < suite_segments )) \ + || fail "$job $suite segment_set entry $token is outside 0..$(( suite_segments - 1 ))" + done + fi if [[ -z "$declared" ]]; then declared=$suite_segments elif [[ "$declared" != "$suite_segments" ]]; then @@ -263,29 +276,66 @@ check_subset() { fail "subset '$set_value' for $suite ran: $actual" } } -check_subset motd 8 '0 1 2 3' 'test-motd.sh-0 +check_subset motd 11 '0 1 2 3' 'test-motd.sh-0 test-motd.sh-1 test-motd.sh-2 test-motd.sh-3' -check_subset motd 8 '' 'test-motd.sh-0 +# Double-digit segments must run as themselves, not as "1" plus a stray "0". +check_subset motd 11 '8 9 10' 'test-motd.sh-8 +test-motd.sh-9 +test-motd.sh-10' +check_subset motd 11 '' 'test-motd.sh-0 test-motd.sh-1 test-motd.sh-2 test-motd.sh-3 test-motd.sh-4 test-motd.sh-5 test-motd.sh-6 -test-motd.sh-7' +test-motd.sh-7 +test-motd.sh-8 +test-motd.sh-9 +test-motd.sh-10' check_subset motd 1 '' 'test-motd.sh' -check_subset cloudflare 8 '4 5 6 7' 'test-cloudflare-tunnel.sh-4 +check_subset cloudflare 12 '2 3 5 6 11' 'test-cloudflare-tunnel.sh-2 +test-cloudflare-tunnel.sh-3 +test-cloudflare-tunnel.sh-5 +test-cloudflare-tunnel.sh-6 +test-cloudflare-tunnel.sh-11' +check_subset cloudflare 12 '0 1 4 7 8 9 10' 'test-cloudflare-tunnel.sh-0 +test-cloudflare-tunnel.sh-1 +test-cloudflare-tunnel.sh-4 +test-cloudflare-tunnel.sh-7 +test-cloudflare-tunnel.sh-8 +test-cloudflare-tunnel.sh-9 +test-cloudflare-tunnel.sh-10' +check_subset cloudflare 12 '' 'test-cloudflare-tunnel.sh-0 +test-cloudflare-tunnel.sh-1 +test-cloudflare-tunnel.sh-2 +test-cloudflare-tunnel.sh-3 +test-cloudflare-tunnel.sh-4 test-cloudflare-tunnel.sh-5 test-cloudflare-tunnel.sh-6 -test-cloudflare-tunnel.sh-7' +test-cloudflare-tunnel.sh-7 +test-cloudflare-tunnel.sh-8 +test-cloudflare-tunnel.sh-9 +test-cloudflare-tunnel.sh-10 +test-cloudflare-tunnel.sh-11' check_subset cloudflare 1 '' 'test-cloudflare-tunnel.sh' -check_subset push-worker 4 '2' 'test-push-worker-registration.sh-2' -check_subset push-worker 4 '' 'test-push-worker-registration.sh-0 +check_subset push-worker 7 '2' 'test-push-worker-registration.sh-2' +check_subset push-worker 7 '0 1 2 3' 'test-push-worker-registration.sh-0 test-push-worker-registration.sh-1 test-push-worker-registration.sh-2 test-push-worker-registration.sh-3' +check_subset push-worker 7 '4 5 6' 'test-push-worker-registration.sh-4 +test-push-worker-registration.sh-5 +test-push-worker-registration.sh-6' +check_subset push-worker 7 '' 'test-push-worker-registration.sh-0 +test-push-worker-registration.sh-1 +test-push-worker-registration.sh-2 +test-push-worker-registration.sh-3 +test-push-worker-registration.sh-4 +test-push-worker-registration.sh-5 +test-push-worker-registration.sh-6' check_subset push-worker 1 '' 'test-push-worker-registration.sh' check_reject() { @@ -299,23 +349,51 @@ check_reject() { # A segment count the suite's source does not have would silently run part of it. check_reject 'motd cut into two' motd 2 '' check_reject 'motd cut into four' motd 4 '' +check_reject 'motd cut into eight' motd 8 '' +check_reject 'motd cut into twelve' motd 12 '' check_reject 'cloudflare cut into four' cloudflare 4 '' +check_reject 'cloudflare cut into eight' cloudflare 8 '' +check_reject 'cloudflare cut into two' cloudflare 2 '' check_reject 'push-worker cut into eight' push-worker 8 '' +check_reject 'push-worker cut into four' push-worker 4 '' check_reject 'push-worker cut into two' push-worker 2 '' -check_reject 'motd segment past its source' motd 8 '8' +check_reject 'motd segment past its source' motd 11 '11' check_reject 'subset for an unsegmented suite' push 2 '0' check_reject 'subset for the other bucket' other 2 '0' check_reject 'subset without segmented mode' motd 1 '0' check_reject 'mixed plan with one count' all 8 '' check_reject 'mixed plan with four segments' all 4 '0 1 2 3' -# Anything but a distinct single-digit in-range index is refused; a file named 1 -# must not turn "[1]" into a real segment. +# Anything but a distinct in-range index of the declared range is refused, and +# the refusal has to come from the segment validation itself: the declared count +# stays untouched, so a count mismatch cannot stand in for a token that got +# through. A file named 1 must not turn "[1]" into a real segment. +segment_rejection='Invalid TEST_SEGMENT_SET entry|Duplicate segment in TEST_SEGMENT_SET|out of range for TEST_SEGMENTS' +check_bad_set() { + local suite=$1 segs=$2 bad=$3 rc=0 + : > "$CAPTURE" + ( cd "$TEMP_DIR" && TEST_SEGMENTS="$segs" TEST_SEGMENT_SET="$bad" \ + bash "$TEMP_DIR/tests/run.sh" "$suite" > "$TEMP_DIR/output" 2>&1 ) || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' for $suite accepted (exit $rc)"; } + grep -qE "$segment_rejection" "$TEMP_DIR/output" \ + || { cat "$TEMP_DIR/output"; fail "subset '$bad' for $suite was not refused by the segment validation"; } +} : > "$TEMP_DIR/1" for bad in '1 01' '9223372036854775808' '18446744073709551616' '[1]' '1,2' '-1' '1 1' '+1' '0x1' 'x'; do - : > "$CAPTURE" - rc=0 - ( cd "$TEMP_DIR" && TEST_SEGMENTS=8 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? - [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted (exit $rc)"; } + check_bad_set motd 11 "$bad" +done +for bad in '10x' '11' '010' '1 10 10' '0 11'; do + check_bad_set motd 11 "$bad" +done +# The seven-segment suite validates its own tokens the same way, so a set that +# only matches a stale segment count cannot pass for a good one. +for bad in '0 1 2 3x' '4 5 6x' '7' '6 6' '+1' '010'; do + check_bad_set push-worker 7 "$bad" +done + +# The twelve-segment suite validates its own tokens the same way, so a set that +# only matches the old eight-segment layout cannot pass for a good one. +for bad in '0 1 2 3x' '0 2 6x' '12' '11 11' '+1' '010'; do + check_bad_set cloudflare 12 "$bad" done for bad in 0 9 08 '4x' '1 4' 9223372036854775808 18446744073709551616; do : > "$CAPTURE"