diff --git a/python/noxfile.py b/python/noxfile.py index 6166b3f93..2209e4fd5 100644 --- a/python/noxfile.py +++ b/python/noxfile.py @@ -116,7 +116,16 @@ def ui_tests(session: nox.Session) -> None: """ session.install("-r", "requirements.txt") session.install("-r", "requirements_dev.txt") - session.run("pytest", "-m", "integration", "tests/test_ui_modules.py") + # A test that runs past the timeout dumps every thread's stack to the log + # (pytest's faulthandler plugin), so a hang in CI shows where it is. + session.run( + "pytest", + "-m", + "integration", + "-o", + "faulthandler_timeout=180", + "tests/test_ui_modules.py", + ) @nox.session(reuse_venv=True, python="3.9") diff --git a/python/tests/test_ui_modules.py b/python/tests/test_ui_modules.py index ff4b83af7..90ddd031f 100644 --- a/python/tests/test_ui_modules.py +++ b/python/tests/test_ui_modules.py @@ -355,11 +355,22 @@ def _no_comet_download(): Stub the network function to report "no download" (success=False); the real catalog/threading logic stays intact and just loads no comets, keeping the suite hermetic. + + A failed download starts CometCatalog's background retry, which loops + "sleep 60 s, download" until the comet file exists. Under `_fast_sleep` + that loop never sleeps and the stubbed download never makes the file, so + it spun for the whole session, calling (and recording) the mock and + logging on every pass, until CI's tests crawled past the job's time + limit. With no network there is nothing to retry, so the retry is off. """ import PiFinder.comets as comets + from PiFinder.comet_catalog import CometCatalog - with mock.patch.object( - comets, "comet_data_download", return_value=(False, None, None) + with ( + mock.patch.object( + comets, "comet_data_download", return_value=(False, None, None) + ), + mock.patch.object(CometCatalog, "_start_background_retry", lambda self: None), ): yield @@ -434,6 +445,13 @@ def camera_image(): def catalogs(_no_comet_download) -> Iterator[Catalogs]: """Build the real catalogs once from the bundled DB. + Without a catalog cache (always so in CI), the build hands the deferred + objects to a background CatalogLoader thread. Wait for it before any test + runs, so every test sees fully loaded catalogs: its completion callback + adds objects to the live catalogs, re-filters them and writes the cache + from that thread, and `_fast_sleep` removes its per-batch yield, so it + would otherwise compete flat out with the first tests. + Teardown stops the perpetual catalog timers. The planet and comet catalogs run a self-rescheduling, *non-daemon* threading.Timer (via TimerMixin); left running they keep the interpreter alive at shutdown, @@ -443,6 +461,11 @@ def catalogs(_no_comet_download) -> Iterator[Catalogs]: boot_state = SharedStateObj() boot_state.set_ui_state(UIState()) built = CatalogBuilder().build(boot_state, queue.Queue()) + loader = getattr(built, "_background_loader", None) + thread = getattr(loader, "_thread", None) + if thread is not None: + thread.join(timeout=600) + assert not thread.is_alive(), "background catalog loading did not finish" yield built for catalog in built.get_catalogs(only_selected=False):