From f9b018a95795d00b8c7af0cccaa844df3f1a076d Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 28 Oct 2026 21:23:39 -0700 Subject: [PATCH 1/2] test(ui): wait for the background catalog loader before the UI tests run ui_tests has hung intermittently on CI since 2026-10-24: four of the last six nox runs (main at 0e0b2b14e and c3559282e, #681, #683) stalled in tests/test_ui_modules.py until the job's 20-minute limit, at different points in the run, while every other session passed. Locally the suite passes in about 80 s, warm or cold cache; CI's passing runs took 150-400 s. The session-scoped catalogs fixture yielded while CatalogBuilder's background CatalogLoader thread was still loading the deferred objects (CI never has a catalog cache). The autouse _fast_sleep fixture no-ops time.sleep, which removes the loader's per-batch yield, so it ran flat out beside every test; and on completion its callback added objects to the live catalogs, re-filtered them and pickled them to the cache, from that thread, while tests were reading the same catalogs. The fixture now joins the loader before yielding, so the tests run on fully loaded catalogs with no thread mutating them. ui_tests also sets pytest's faulthandler_timeout, so a test still running after 180 s dumps every thread's stack to the log instead of the job dying silently. Co-Authored-By: Claude Opus 5.5 --- python/noxfile.py | 11 ++++++++++- python/tests/test_ui_modules.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/python/noxfile.py b/python/noxfile.py index 6166b3f93..2209e4fd5 100644 --- a/python/noxfile.py +++ b/python/noxfile.py @@ -116,7 +116,16 @@ def ui_tests(session: nox.Session) -> None: """ session.install("-r", "requirements.txt") session.install("-r", "requirements_dev.txt") - session.run("pytest", "-m", "integration", "tests/test_ui_modules.py") + # A test that runs past the timeout dumps every thread's stack to the log + # (pytest's faulthandler plugin), so a hang in CI shows where it is. + session.run( + "pytest", + "-m", + "integration", + "-o", + "faulthandler_timeout=180", + "tests/test_ui_modules.py", + ) @nox.session(reuse_venv=True, python="3.9") diff --git a/python/tests/test_ui_modules.py b/python/tests/test_ui_modules.py index ff4b83af7..113cb7552 100644 --- a/python/tests/test_ui_modules.py +++ b/python/tests/test_ui_modules.py @@ -434,6 +434,13 @@ def camera_image(): def catalogs(_no_comet_download) -> Iterator[Catalogs]: """Build the real catalogs once from the bundled DB. + Without a catalog cache (always so in CI), the build hands the deferred + objects to a background CatalogLoader thread. Wait for it before any test + runs: its completion callback adds objects to the live catalogs and + re-filters them, and writes the cache, from that thread, so tests racing + it saw catalogs change under them. `_fast_sleep` also removes the loader's + per-batch yield, so it would otherwise compete flat out with every test. + Teardown stops the perpetual catalog timers. The planet and comet catalogs run a self-rescheduling, *non-daemon* threading.Timer (via TimerMixin); left running they keep the interpreter alive at shutdown, @@ -443,6 +450,11 @@ def catalogs(_no_comet_download) -> Iterator[Catalogs]: boot_state = SharedStateObj() boot_state.set_ui_state(UIState()) built = CatalogBuilder().build(boot_state, queue.Queue()) + loader = getattr(built, "_background_loader", None) + thread = getattr(loader, "_thread", None) + if thread is not None: + thread.join(timeout=600) + assert not thread.is_alive(), "background catalog loading did not finish" yield built for catalog in built.get_catalogs(only_selected=False): From d315b6d201f9942a4b8ce9262bde1ced22462fe6 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 28 Oct 2026 21:48:11 -0700 Subject: [PATCH 2/2] test(ui): stop the comet catalog's retry loop spinning through the UI tests The CI thread dump from the previous commit's faulthandler_timeout found the hang. A failed comet download starts CometCatalog's background retry, which loops "sleep 60 s, download" until the comet file exists. In the UI harness the download is stubbed to fail and _fast_sleep no-ops time.sleep, so that loop never slept and never ended: it spun for the whole session, calling and recording the mock and logging on every pass, until CI's tests crawled past the job's 20-minute limit. A machine that already has a comet file in ~/PiFinder_data never starts the loop, which is why it didn't reproduce locally. Run with an empty home, a four-test slice logged 5,012 retries; with the retry stubbed out, none. _no_comet_download now stubs _start_background_retry too: with no network there is nothing to retry. The loader join from the previous commit stays, reworded: it isn't the cause, but it keeps the loader from competing with the first tests and every test sees fully loaded catalogs. Co-Authored-By: Claude Opus 5.5 --- python/tests/test_ui_modules.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/python/tests/test_ui_modules.py b/python/tests/test_ui_modules.py index 113cb7552..90ddd031f 100644 --- a/python/tests/test_ui_modules.py +++ b/python/tests/test_ui_modules.py @@ -355,11 +355,22 @@ def _no_comet_download(): Stub the network function to report "no download" (success=False); the real catalog/threading logic stays intact and just loads no comets, keeping the suite hermetic. + + A failed download starts CometCatalog's background retry, which loops + "sleep 60 s, download" until the comet file exists. Under `_fast_sleep` + that loop never sleeps and the stubbed download never makes the file, so + it spun for the whole session, calling (and recording) the mock and + logging on every pass, until CI's tests crawled past the job's time + limit. With no network there is nothing to retry, so the retry is off. """ import PiFinder.comets as comets + from PiFinder.comet_catalog import CometCatalog - with mock.patch.object( - comets, "comet_data_download", return_value=(False, None, None) + with ( + mock.patch.object( + comets, "comet_data_download", return_value=(False, None, None) + ), + mock.patch.object(CometCatalog, "_start_background_retry", lambda self: None), ): yield @@ -436,10 +447,10 @@ def catalogs(_no_comet_download) -> Iterator[Catalogs]: Without a catalog cache (always so in CI), the build hands the deferred objects to a background CatalogLoader thread. Wait for it before any test - runs: its completion callback adds objects to the live catalogs and - re-filters them, and writes the cache, from that thread, so tests racing - it saw catalogs change under them. `_fast_sleep` also removes the loader's - per-batch yield, so it would otherwise compete flat out with every test. + runs, so every test sees fully loaded catalogs: its completion callback + adds objects to the live catalogs, re-filters them and writes the cache + from that thread, and `_fast_sleep` removes its per-batch yield, so it + would otherwise compete flat out with the first tests. Teardown stops the perpetual catalog timers. The planet and comet catalogs run a self-rescheduling, *non-daemon* threading.Timer (via TimerMixin);