Test verify-tap.sh, the standalone TAP output checker

CLAUDE.md

@@ -223,8 +223,8 @@ section once resolved.
    misleading prefix is judged lower-risk than touching it.
 4. **Scripts a user is realistically expected to invoke by hand**:
    `setup.sh`, `pgxntool-sync.sh`, `update-setup-files.sh`, `bin/version`,
-   and `pgtle.sh` (including its own CLI flags, not just the make targets
-   that wrap it).
+   `verify-tap.sh`, and `pgtle.sh` (including its own CLI flags, not just
+   the make targets that wrap it).
 5. **`DEBUG` is a special case**: its existence may be documented (it's
    fine for users to know it exists), but the specific level numbers are
    an internal implementation detail, not a documented contract — changing

test/lib/dist-expected-files.txt

@@ -84,6 +84,7 @@ pgxntool/setup.sh
 pgxntool/pgxntool-sync.sh
 pgxntool/update-setup-files.sh
 pgxntool/verify-results-pgtap.sh
+pgxntool/verify-tap.sh
 pgxntool/WHAT_IS_THIS
 
 # pgxntool test infrastructure

test/standard/make-test.bats

@@ -21,6 +21,8 @@
 #   brand-new test's first expected output through (issue #119), and can be
 #   disabled. Its own pass/fail logic lives in verify-results-pgtap-script.bats
 # - PGXNTOOL_VERIFY_RESULTS_MODE is validated (pgtap/diffs, case-insensitive)
+#   and, in pgtap mode, makes `make test` run verify-tap.sh, so a blessed
+#   `not ok` still fails it
 
 load ../lib/helpers
 
@@ -671,7 +673,7 @@ EOF
   done
 }
 
-@test "mixed-case PGXNTOOL_VERIFY_RESULTS_MODE selects the matching verify-results recipe" {
+@test "mixed-case PGXNTOOL_VERIFY_RESULTS_MODE selects the matching verify-results and test recipes" {
   # The pgtap recipe invokes verify-results-pgtap.sh; the diffs recipe is an
   # inline regression.diffs check carrying the block message.
   run make -n verify-results PGXNTOOL_VERIFY_RESULTS_MODE=pgTap 2>&1
@@ -683,6 +685,14 @@ EOF
   assert_success
   assert_contains "$output" "Cannot run 'make results'"
   assert_not_contains "$output" "verify-results-pgtap.sh"
+
+  # `make test` runs verify-tap.sh only in pgtap mode, the default.
+  run make -n test 2>&1
+  assert_success
+  assert_contains "$output" "verify-tap.sh --skip-non-tap"
+  run make -n test PGXNTOOL_VERIFY_RESULTS_MODE=DIFFS 2>&1
+  assert_success
+  assert_not_contains "$output" "verify-tap.sh"
 }
 
 @test "make results updates expected output" {
@@ -753,6 +763,25 @@ EOF
   rm -f test/results/pgtap_fail.out
 }
 
+@test "make test fails on a not ok that was blessed into its expected output" {
+  skip_if_no_postgres
+
+  # A blessed failure matches its expected file, so pg_regress reports
+  # nothing; only make test's own verify-tap.sh run can catch it.
+  sed 's/, 3$/, 4/' test/sql/pgxntool-test.sql > test/sql/blessed_not_ok.sql
+  run make results PGXNTOOL_ENABLE_VERIFY_RESULTS=no
+  assert_success
+  grep -q '^not ok 1' test/expected/blessed_not_ok.out \
+    || error "blessed_not_ok.out holds no 'not ok' line: $(cat test/expected/blessed_not_ok.out)"
+
+  run make test
+  assert_failure
+  assert_contains "$output" "pgtap failure detected in test/results/blessed_not_ok.out"
+  assert_file_not_exists test/regression.diffs
+
+  rm -f test/sql/blessed_not_ok.sql test/expected/blessed_not_ok.out test/results/blessed_not_ok.out
+}
+
 # ============================================================================
 # results-build (issue #108)
 # ============================================================================

test/standard/verify-results-pgtap-script.bats

@@ -110,38 +110,40 @@ ok 1'
 }
 
 @test "verify-results-pgtap.sh: refuses to bless a brand-new test whose output holds a SQL error" {
-  # ON_ERROR_STOP aborts the script at the error, so pgtap emits neither
-  # 'not ok' nor its plan-mismatch line and the scan above finds nothing.
+  # An error after the last planned test leaves the TAP itself intact, so
+  # verify-tap.sh passes it and only the regression.diffs scan can catch it.
   #
   # Both of psql's real renderings have to block. pg_regress feeds psql on
   # stdin, so an error in the test file itself starts at column 0, while one
   # inside an \i'd file (setup.sql/finish.sql, which every pgxntool test
   # sources) carries a "psql:<file>:<line>: " prefix -- a column-anchored
   # match would sail straight past the second.
-  write_results errtest '1..2
+  write_results errtest '1..1
 ok 1
 ERROR:  42P01: relation "nope" does not exist'
   append_diff_block errtest '@@ -0,0 +1,3 @@
-+1..2
++1..1
 +ok 1
 +ERROR:  42P01: relation "nope" does not exist'
 
   run "$SCRIPT" "$TESTOUT"
   assert_failure_with_status 1
   assert_contains "$output" "errtest.out"
+  assert_contains "$output" "Tests are failing"
 
   rm -f "$TESTOUT/regression.diffs"
-  write_results errtest '1..2
+  write_results errtest '1..1
 ok 1
 psql:test/pgxntool/setup.sql:3: ERROR:  42P01: relation "nope" does not exist'
   append_diff_block errtest '@@ -0,0 +1,3 @@
-+1..2
++1..1
 +ok 1
 +psql:test/pgxntool/setup.sql:3: ERROR:  42P01: relation "nope" does not exist'
 
   run "$SCRIPT" "$TESTOUT"
   assert_failure_with_status 1
   assert_contains "$output" "errtest.out"
+  assert_contains "$output" "Tests are failing"
 }
 
 @test "verify-results-pgtap.sh: blesses a passing test whose description merely mentions ERROR:" {
@@ -310,6 +312,22 @@ ok 2
   assert_contains "$output" "pgtap plan mismatch"
 }
 
+@test "verify-results-pgtap.sh: blocks pgtap output with no plan line, leaving plain-SQL output alone" {
+  # A no_plan() test that died before finish(): no 'not ok', no mismatch line.
+  write_results died 'ok 1
+ok 2'
+  write_results plain ' x
+---
+ 1
+(1 row)'
+
+  run "$SCRIPT" "$TESTOUT"
+  assert_failure_with_status 1
+  assert_contains "$output" "died.out has no plan line"
+  assert_contains "$output" "Cannot run 'make results'"
+  assert_not_contains "$output" "plain.out"
+}
+
 @test "verify-results-pgtap.sh: usage error when called with no TESTOUT argument" {
   run "$SCRIPT"
   assert_failure

test/standard/verify-tap-script.bats

@@ -0,0 +1,215 @@
+#!/usr/bin/env bats
+
+# Test: verify-tap.sh - pure script-logic unit tests
+#
+# These tests invoke pgxntool's verify-tap.sh directly on scratch files and
+# stdin -- no foundation environment, no `make`, no PostgreSQL. They own all
+# of its decision logic: which TAP streams fail, how inputs are named, and
+# its exit codes. verify-results-pgtap-script.bats covers
+# verify-results-pgtap.sh's use of this script.
+
+load ../lib/helpers
+load ../lib/assertions
+
+setup_file() {
+  setup_topdir
+  load_test_env "verify-tap-script"  # only resolves PGXNREPO; creates no foundation
+}
+
+setup() {
+  load_test_env "verify-tap-script"  # only resolves PGXNREPO; creates no foundation
+  export SCRIPT="$PGXNREPO/verify-tap.sh"
+  export OUTDIR="$BATS_TEST_TMPDIR/out"
+  mkdir -p "$OUTDIR"
+}
+
+# Write $2 as the content of $OUTDIR/$1.out.
+write_out() {
+  printf '%s\n' "$2" > "$OUTDIR/$1.out"
+}
+
+@test "verify-tap.sh: passes clean TAP, plan last, a skip_all 1..0, TODO failures and subtests" {
+  write_out clean '\set ECHO none
+1..2
+ok 1
+ok 2 - described
+# TRANSACTION INTENTIONALLY LEFT OPEN!'
+  write_out planlast 'ok 1
+ok 2 - described
+1..2'
+  write_out skipall '1..0 # Skipped: nothing to do here'
+  write_out todo '1..2
+ok 1
+not ok 2 - known issue # TODO fix later'
+  # Indented subtest lines count toward nothing, failures included.
+  write_out subtest '1..1
+    1..2
+    ok 1
+    not ok 2
+    Bail out!
+ok 1 - subtest'
+
+  run "$SCRIPT" "$OUTDIR/clean.out" "$OUTDIR/planlast.out" "$OUTDIR/skipall.out" \
+    "$OUTDIR/todo.out" "$OUTDIR/subtest.out"
+  assert_success
+  [ -z "$output" ] || error "expected no output, got: $output"
+}
+
+@test "verify-tap.sh: fails on not ok, Bail out! and pgTAP's 'Looks like you planned' line" {
+  write_out failing '1..2
+ok 1
+not ok 2 - broken'
+  run "$SCRIPT" "$OUTDIR/failing.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "pgtap failure detected in $OUTDIR/failing.out"
+  assert_contains "$output" "not ok 2 - broken"
+
+  write_out bail '1..2
+ok 1
+Bail out! database went away'
+  run "$SCRIPT" "$OUTDIR/bail.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "bailed out in $OUTDIR/bail.out"
+  assert_contains "$output" "Bail out! database went away"
+
+  write_out planned '1..3
+ok 1
+ok 2
+# Looks like you planned 3 tests but ran 2'
+  run "$SCRIPT" "$OUTDIR/planned.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "pgtap plan mismatch in $OUTDIR/planned.out"
+  assert_contains "$output" "Looks like you planned 3 tests but ran 2"
+  # pgTAP's own line already says it; no second count message.
+  assert_not_contains "$output" "but found"
+}
+
+@test "verify-tap.sh: fails without exactly one plan line, placed first or last" {
+  # A no_plan() test that died before finish() printed its plan.
+  write_out noplan 'ok 1
+ok 2'
+  run "$SCRIPT" "$OUTDIR/noplan.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "$OUTDIR/noplan.out has no plan line"
+
+  write_out twoplans '1..1
+ok 1
+1..1'
+  run "$SCRIPT" "$OUTDIR/twoplans.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "$OUTDIR/twoplans.out has 2 plan lines"
+
+  write_out middle 'ok 1
+1..2
+ok 2'
+  run "$SCRIPT" "$OUTDIR/middle.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "$OUTDIR/middle.out has its plan line (1..N) between test lines"
+}
+
+@test "verify-tap.sh: fails when the test count differs from the plan" {
+  # A plan(N) test that died partway through: no pgTAP mismatch line at all.
+  write_out short '1..3
+ok 1
+ERROR:  boom'
+  write_out long '1..1
+ok 1
+ok 2'
+
+  run "$SCRIPT" "$OUTDIR/short.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "plan mismatch in $OUTDIR/short.out: planned 3 tests but found 1"
+
+  run "$SCRIPT" "$OUTDIR/long.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "planned 1 tests but found 2"
+}
+
+@test "verify-tap.sh: fails on input with no TAP at all, unless --skip-non-tap skips it" {
+  # Plain SQL output, even mentioning TAP phrases mid-line, means nothing ran.
+  write_out plain '\set ECHO none
+
+ x
+---
+ not ok here
+(1 row)'
+
+  run "$SCRIPT" "$OUTDIR/plain.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "no TAP output in $OUTDIR/plain.out"
+
+  run "$SCRIPT" --skip-non-tap "$OUTDIR/plain.out"
+  assert_success
+  [ -z "$output" ] || error "expected no output, got: $output"
+
+  # Skipping is only for input with no TAP lines; a TAP stream beside it is
+  # still judged, wherever the flag goes, and so is stdin.
+  write_out bad '1..1
+not ok 1'
+  run "$SCRIPT" "$OUTDIR/plain.out" "$OUTDIR/bad.out" --skip-non-tap
+  assert_failure_with_status 1
+  assert_contains "$output" "pgtap failure detected in $OUTDIR/bad.out"
+  assert_not_contains "$output" "plain.out"
+
+  run bash -c '"$SCRIPT" --skip-non-tap < "$OUTDIR/plain.out"'
+  assert_success
+  run bash -c '"$SCRIPT" --skip-non-tap < "$OUTDIR/bad.out"'
+  assert_failure_with_status 1
+}
+
+@test "verify-tap.sh: reads stdin with no arguments or '-', reporting to stderr as stdin" {
+  run bash -c 'printf "1..1\nok 1\n" | "$SCRIPT"'
+  assert_success
+
+  write_out bad '1..1
+not ok 1'
+  run bash -c '"$SCRIPT" < "$OUTDIR/bad.out"'
+  assert_failure_with_status 1
+  assert_contains "$output" "pgtap failure detected in stdin"
+
+  # `-` mixes with files; every diagnostic goes to stderr, none to stdout.
+  write_out good '1..1
+ok 1'
+  run bash -c '"$SCRIPT" "$OUTDIR/good.out" - < "$OUTDIR/bad.out" 2>/dev/null'
+  assert_failure_with_status 1
+  [ -z "$output" ] || error "expected nothing on stdout, got: $output"
+  run bash -c '"$SCRIPT" "$OUTDIR/good.out" - < "$OUTDIR/bad.out" 2>&1 >/dev/null'
+  assert_contains "$output" "pgtap failure detected in stdin"
+}
+
+@test "verify-tap.sh: reports every failing file and fails if any one fails" {
+  write_out good '1..1
+ok 1'
+  write_out bad1 '1..1
+not ok 1'
+  write_out bad2 'ok 1'
+
+  run "$SCRIPT" "$OUTDIR/good.out" "$OUTDIR/bad1.out" "$OUTDIR/bad2.out"
+  assert_failure_with_status 1
+  assert_contains "$output" "bad1.out"
+  assert_contains "$output" "bad2.out"
+  assert_not_contains "$output" "good.out"
+}
+
+@test "verify-tap.sh: exits 2 on a usage error or an unreadable file, judging nothing" {
+  write_out bad '1..1
+not ok 1'
+
+  run "$SCRIPT" "$OUTDIR/bad.out" "$OUTDIR/missing.out"
+  assert_failure_with_status 2
+  assert_contains "$output" "cannot read '$OUTDIR/missing.out'"
+  assert_not_contains "$output" "pgtap failure"
+
+  run "$SCRIPT" "$OUTDIR"
+  assert_failure_with_status 2
+
+  run "$SCRIPT" --bogus "$OUTDIR/bad.out"
+  assert_failure_with_status 2
+  assert_contains "$output" "Usage:"
+
+  run "$SCRIPT" --help
+  assert_success
+  assert_contains "$output" "Usage:"
+}
+
+# vi: expandtab ts=2 sw=2