diff --git a/CLAUDE.md b/CLAUDE.md index e2ebd94..f8dad42 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -223,8 +223,8 @@ section once resolved. misleading prefix is judged lower-risk than touching it. 4. **Scripts a user is realistically expected to invoke by hand**: `setup.sh`, `pgxntool-sync.sh`, `update-setup-files.sh`, `bin/version`, - and `pgtle.sh` (including its own CLI flags, not just the make targets - that wrap it). + `verify-tap.sh`, and `pgtle.sh` (including its own CLI flags, not just + the make targets that wrap it). 5. **`DEBUG` is a special case**: its existence may be documented (it's fine for users to know it exists), but the specific level numbers are an internal implementation detail, not a documented contract — changing diff --git a/test/lib/dist-expected-files.txt b/test/lib/dist-expected-files.txt index b6e21b3..36e80d3 100644 --- a/test/lib/dist-expected-files.txt +++ b/test/lib/dist-expected-files.txt @@ -84,6 +84,7 @@ pgxntool/setup.sh pgxntool/pgxntool-sync.sh pgxntool/update-setup-files.sh pgxntool/verify-results-pgtap.sh +pgxntool/verify-tap.sh pgxntool/WHAT_IS_THIS # pgxntool test infrastructure diff --git a/test/standard/make-test.bats b/test/standard/make-test.bats index 9ab4169..c86a334 100755 --- a/test/standard/make-test.bats +++ b/test/standard/make-test.bats @@ -21,6 +21,8 @@ # brand-new test's first expected output through (issue #119), and can be # disabled. Its own pass/fail logic lives in verify-results-pgtap-script.bats # - PGXNTOOL_VERIFY_RESULTS_MODE is validated (pgtap/diffs, case-insensitive) +# and, in pgtap mode, makes `make test` run verify-tap.sh, so a blessed +# `not ok` still fails it load ../lib/helpers @@ -671,7 +673,7 @@ EOF done } -@test "mixed-case PGXNTOOL_VERIFY_RESULTS_MODE selects the matching verify-results recipe" { +@test "mixed-case PGXNTOOL_VERIFY_RESULTS_MODE selects the matching verify-results and test recipes" { # The pgtap recipe invokes verify-results-pgtap.sh; the diffs recipe is an # inline regression.diffs check carrying the block message. run make -n verify-results PGXNTOOL_VERIFY_RESULTS_MODE=pgTap 2>&1 @@ -683,6 +685,14 @@ EOF assert_success assert_contains "$output" "Cannot run 'make results'" assert_not_contains "$output" "verify-results-pgtap.sh" + + # `make test` runs verify-tap.sh only in pgtap mode, the default. + run make -n test 2>&1 + assert_success + assert_contains "$output" "verify-tap.sh --skip-non-tap" + run make -n test PGXNTOOL_VERIFY_RESULTS_MODE=DIFFS 2>&1 + assert_success + assert_not_contains "$output" "verify-tap.sh" } @test "make results updates expected output" { @@ -753,6 +763,25 @@ EOF rm -f test/results/pgtap_fail.out } +@test "make test fails on a not ok that was blessed into its expected output" { + skip_if_no_postgres + + # A blessed failure matches its expected file, so pg_regress reports + # nothing; only make test's own verify-tap.sh run can catch it. + sed 's/, 3$/, 4/' test/sql/pgxntool-test.sql > test/sql/blessed_not_ok.sql + run make results PGXNTOOL_ENABLE_VERIFY_RESULTS=no + assert_success + grep -q '^not ok 1' test/expected/blessed_not_ok.out \ + || error "blessed_not_ok.out holds no 'not ok' line: $(cat test/expected/blessed_not_ok.out)" + + run make test + assert_failure + assert_contains "$output" "pgtap failure detected in test/results/blessed_not_ok.out" + assert_file_not_exists test/regression.diffs + + rm -f test/sql/blessed_not_ok.sql test/expected/blessed_not_ok.out test/results/blessed_not_ok.out +} + # ============================================================================ # results-build (issue #108) # ============================================================================ diff --git a/test/standard/verify-results-pgtap-script.bats b/test/standard/verify-results-pgtap-script.bats index def95b3..693baa1 100644 --- a/test/standard/verify-results-pgtap-script.bats +++ b/test/standard/verify-results-pgtap-script.bats @@ -110,38 +110,40 @@ ok 1' } @test "verify-results-pgtap.sh: refuses to bless a brand-new test whose output holds a SQL error" { - # ON_ERROR_STOP aborts the script at the error, so pgtap emits neither - # 'not ok' nor its plan-mismatch line and the scan above finds nothing. + # An error after the last planned test leaves the TAP itself intact, so + # verify-tap.sh passes it and only the regression.diffs scan can catch it. # # Both of psql's real renderings have to block. pg_regress feeds psql on # stdin, so an error in the test file itself starts at column 0, while one # inside an \i'd file (setup.sql/finish.sql, which every pgxntool test # sources) carries a "psql::: " prefix -- a column-anchored # match would sail straight past the second. - write_results errtest '1..2 + write_results errtest '1..1 ok 1 ERROR: 42P01: relation "nope" does not exist' append_diff_block errtest '@@ -0,0 +1,3 @@ -+1..2 ++1..1 +ok 1 +ERROR: 42P01: relation "nope" does not exist' run "$SCRIPT" "$TESTOUT" assert_failure_with_status 1 assert_contains "$output" "errtest.out" + assert_contains "$output" "Tests are failing" rm -f "$TESTOUT/regression.diffs" - write_results errtest '1..2 + write_results errtest '1..1 ok 1 psql:test/pgxntool/setup.sql:3: ERROR: 42P01: relation "nope" does not exist' append_diff_block errtest '@@ -0,0 +1,3 @@ -+1..2 ++1..1 +ok 1 +psql:test/pgxntool/setup.sql:3: ERROR: 42P01: relation "nope" does not exist' run "$SCRIPT" "$TESTOUT" assert_failure_with_status 1 assert_contains "$output" "errtest.out" + assert_contains "$output" "Tests are failing" } @test "verify-results-pgtap.sh: blesses a passing test whose description merely mentions ERROR:" { @@ -310,6 +312,22 @@ ok 2 assert_contains "$output" "pgtap plan mismatch" } +@test "verify-results-pgtap.sh: blocks pgtap output with no plan line, leaving plain-SQL output alone" { + # A no_plan() test that died before finish(): no 'not ok', no mismatch line. + write_results died 'ok 1 +ok 2' + write_results plain ' x +--- + 1 +(1 row)' + + run "$SCRIPT" "$TESTOUT" + assert_failure_with_status 1 + assert_contains "$output" "died.out has no plan line" + assert_contains "$output" "Cannot run 'make results'" + assert_not_contains "$output" "plain.out" +} + @test "verify-results-pgtap.sh: usage error when called with no TESTOUT argument" { run "$SCRIPT" assert_failure diff --git a/test/standard/verify-tap-script.bats b/test/standard/verify-tap-script.bats new file mode 100644 index 0000000..53f4a93 --- /dev/null +++ b/test/standard/verify-tap-script.bats @@ -0,0 +1,215 @@ +#!/usr/bin/env bats + +# Test: verify-tap.sh - pure script-logic unit tests +# +# These tests invoke pgxntool's verify-tap.sh directly on scratch files and +# stdin -- no foundation environment, no `make`, no PostgreSQL. They own all +# of its decision logic: which TAP streams fail, how inputs are named, and +# its exit codes. verify-results-pgtap-script.bats covers +# verify-results-pgtap.sh's use of this script. + +load ../lib/helpers +load ../lib/assertions + +setup_file() { + setup_topdir + load_test_env "verify-tap-script" # only resolves PGXNREPO; creates no foundation +} + +setup() { + load_test_env "verify-tap-script" # only resolves PGXNREPO; creates no foundation + export SCRIPT="$PGXNREPO/verify-tap.sh" + export OUTDIR="$BATS_TEST_TMPDIR/out" + mkdir -p "$OUTDIR" +} + +# Write $2 as the content of $OUTDIR/$1.out. +write_out() { + printf '%s\n' "$2" > "$OUTDIR/$1.out" +} + +@test "verify-tap.sh: passes clean TAP, plan last, a skip_all 1..0, TODO failures and subtests" { + write_out clean '\set ECHO none +1..2 +ok 1 +ok 2 - described +# TRANSACTION INTENTIONALLY LEFT OPEN!' + write_out planlast 'ok 1 +ok 2 - described +1..2' + write_out skipall '1..0 # Skipped: nothing to do here' + write_out todo '1..2 +ok 1 +not ok 2 - known issue # TODO fix later' + # Indented subtest lines count toward nothing, failures included. + write_out subtest '1..1 + 1..2 + ok 1 + not ok 2 + Bail out! +ok 1 - subtest' + + run "$SCRIPT" "$OUTDIR/clean.out" "$OUTDIR/planlast.out" "$OUTDIR/skipall.out" \ + "$OUTDIR/todo.out" "$OUTDIR/subtest.out" + assert_success + [ -z "$output" ] || error "expected no output, got: $output" +} + +@test "verify-tap.sh: fails on not ok, Bail out! and pgTAP's 'Looks like you planned' line" { + write_out failing '1..2 +ok 1 +not ok 2 - broken' + run "$SCRIPT" "$OUTDIR/failing.out" + assert_failure_with_status 1 + assert_contains "$output" "pgtap failure detected in $OUTDIR/failing.out" + assert_contains "$output" "not ok 2 - broken" + + write_out bail '1..2 +ok 1 +Bail out! database went away' + run "$SCRIPT" "$OUTDIR/bail.out" + assert_failure_with_status 1 + assert_contains "$output" "bailed out in $OUTDIR/bail.out" + assert_contains "$output" "Bail out! database went away" + + write_out planned '1..3 +ok 1 +ok 2 +# Looks like you planned 3 tests but ran 2' + run "$SCRIPT" "$OUTDIR/planned.out" + assert_failure_with_status 1 + assert_contains "$output" "pgtap plan mismatch in $OUTDIR/planned.out" + assert_contains "$output" "Looks like you planned 3 tests but ran 2" + # pgTAP's own line already says it; no second count message. + assert_not_contains "$output" "but found" +} + +@test "verify-tap.sh: fails without exactly one plan line, placed first or last" { + # A no_plan() test that died before finish() printed its plan. + write_out noplan 'ok 1 +ok 2' + run "$SCRIPT" "$OUTDIR/noplan.out" + assert_failure_with_status 1 + assert_contains "$output" "$OUTDIR/noplan.out has no plan line" + + write_out twoplans '1..1 +ok 1 +1..1' + run "$SCRIPT" "$OUTDIR/twoplans.out" + assert_failure_with_status 1 + assert_contains "$output" "$OUTDIR/twoplans.out has 2 plan lines" + + write_out middle 'ok 1 +1..2 +ok 2' + run "$SCRIPT" "$OUTDIR/middle.out" + assert_failure_with_status 1 + assert_contains "$output" "$OUTDIR/middle.out has its plan line (1..N) between test lines" +} + +@test "verify-tap.sh: fails when the test count differs from the plan" { + # A plan(N) test that died partway through: no pgTAP mismatch line at all. + write_out short '1..3 +ok 1 +ERROR: boom' + write_out long '1..1 +ok 1 +ok 2' + + run "$SCRIPT" "$OUTDIR/short.out" + assert_failure_with_status 1 + assert_contains "$output" "plan mismatch in $OUTDIR/short.out: planned 3 tests but found 1" + + run "$SCRIPT" "$OUTDIR/long.out" + assert_failure_with_status 1 + assert_contains "$output" "planned 1 tests but found 2" +} + +@test "verify-tap.sh: fails on input with no TAP at all, unless --skip-non-tap skips it" { + # Plain SQL output, even mentioning TAP phrases mid-line, means nothing ran. + write_out plain '\set ECHO none + + x +--- + not ok here +(1 row)' + + run "$SCRIPT" "$OUTDIR/plain.out" + assert_failure_with_status 1 + assert_contains "$output" "no TAP output in $OUTDIR/plain.out" + + run "$SCRIPT" --skip-non-tap "$OUTDIR/plain.out" + assert_success + [ -z "$output" ] || error "expected no output, got: $output" + + # Skipping is only for input with no TAP lines; a TAP stream beside it is + # still judged, wherever the flag goes, and so is stdin. + write_out bad '1..1 +not ok 1' + run "$SCRIPT" "$OUTDIR/plain.out" "$OUTDIR/bad.out" --skip-non-tap + assert_failure_with_status 1 + assert_contains "$output" "pgtap failure detected in $OUTDIR/bad.out" + assert_not_contains "$output" "plain.out" + + run bash -c '"$SCRIPT" --skip-non-tap < "$OUTDIR/plain.out"' + assert_success + run bash -c '"$SCRIPT" --skip-non-tap < "$OUTDIR/bad.out"' + assert_failure_with_status 1 +} + +@test "verify-tap.sh: reads stdin with no arguments or '-', reporting to stderr as stdin" { + run bash -c 'printf "1..1\nok 1\n" | "$SCRIPT"' + assert_success + + write_out bad '1..1 +not ok 1' + run bash -c '"$SCRIPT" < "$OUTDIR/bad.out"' + assert_failure_with_status 1 + assert_contains "$output" "pgtap failure detected in stdin" + + # `-` mixes with files; every diagnostic goes to stderr, none to stdout. + write_out good '1..1 +ok 1' + run bash -c '"$SCRIPT" "$OUTDIR/good.out" - < "$OUTDIR/bad.out" 2>/dev/null' + assert_failure_with_status 1 + [ -z "$output" ] || error "expected nothing on stdout, got: $output" + run bash -c '"$SCRIPT" "$OUTDIR/good.out" - < "$OUTDIR/bad.out" 2>&1 >/dev/null' + assert_contains "$output" "pgtap failure detected in stdin" +} + +@test "verify-tap.sh: reports every failing file and fails if any one fails" { + write_out good '1..1 +ok 1' + write_out bad1 '1..1 +not ok 1' + write_out bad2 'ok 1' + + run "$SCRIPT" "$OUTDIR/good.out" "$OUTDIR/bad1.out" "$OUTDIR/bad2.out" + assert_failure_with_status 1 + assert_contains "$output" "bad1.out" + assert_contains "$output" "bad2.out" + assert_not_contains "$output" "good.out" +} + +@test "verify-tap.sh: exits 2 on a usage error or an unreadable file, judging nothing" { + write_out bad '1..1 +not ok 1' + + run "$SCRIPT" "$OUTDIR/bad.out" "$OUTDIR/missing.out" + assert_failure_with_status 2 + assert_contains "$output" "cannot read '$OUTDIR/missing.out'" + assert_not_contains "$output" "pgtap failure" + + run "$SCRIPT" "$OUTDIR" + assert_failure_with_status 2 + + run "$SCRIPT" --bogus "$OUTDIR/bad.out" + assert_failure_with_status 2 + assert_contains "$output" "Usage:" + + run "$SCRIPT" --help + assert_success + assert_contains "$output" "Usage:" +} + +# vi: expandtab ts=2 sw=2