Serve hgvs UTA lookups from a recorded cache in tests

.gitignore

@@ -158,3 +158,6 @@ notebooks/scratch/
 notes
 lint
 uta_*.pgd.gz
+
+# partial hgvs cache from an interrupted `make record-hgvs-cache`
+tests/data/hgvs_cache.pkl.new

Makefile

@@ -128,6 +128,14 @@ cleanest: cleaner
 clean-cassettes:
 	find ./tests -type f -path '*/cassettes/*.yaml' -print0 | ${XRM}
 
+#=> record-hgvs-cache: re-record tests/data/hgvs_cache.json (needs UTA_DB_URL and a running seqrepo-rest-service at SEQREPO_REST_URL; VCR cassettes are not touched)
+.PHONY: record-hgvs-cache
+record-hgvs-cache:
+	rm -f tests/data/hgvs_cache.pkl.new
+	HGVS_SEQREPO_URL="$${SEQREPO_REST_URL:-http://localhost:5000/seqrepo}" VRS_HGVS_CACHE_MODE=learn VRS_HGVS_CACHE_FILE=$(CURDIR)/tests/data/hgvs_cache.pkl.new pytest --disable-vcr
+	python tests/hgvs_cache_json.py tests/data/hgvs_cache.pkl.new tests/data/hgvs_cache.json
+	rm tests/data/hgvs_cache.pkl.new
+
 
 ## <LICENSE>
 ## Copyright 2016 Source Code Committers

README.md

@@ -315,6 +315,41 @@ To run tests:
 make test
 ```
 
+#### Recorded test data
+
+Tests run offline by replaying recorded data, so they don't need a SeqRepo REST
+service, a UTA database, or network access. Set `SEQREPO_ROOT_DIR=tests/data/seqrepo/latest`
+(as CI does) when running or recording tests.
+
+- **VCR cassettes** (`tests/**/cassettes/*.yaml`) replay vrs-python's HTTP requests to
+  the [SeqRepo REST service](https://github.com/biocommons/seqrepo-rest-service). CI
+  runs `pytest --vcr-record=none`, so a test making an unrecorded request fails.
+- **The hgvs cache** (`tests/data/hgvs_cache.json`) replays the hgvs library's UTA
+  queries and sequence fetches. `tests/conftest.py` enables it by default
+  (`VRS_HGVS_CACHE_MODE=run`), and a lookup missing from the cache raises
+  `HGVSDataNotAvailableError`. hgvs itself reads and writes its cache as a pickle;
+  `tests/hgvs_cache_json.py` converts between the two.
+
+To re-record, start the external services (`docker compose up` starts
+seqrepo-rest-service on port 5000 and UTA on port 5432). With an existing local SeqRepo
+snapshot, you can instead run just the REST service with it mounted:
+`docker run -p 5000:5000 -v /path/to/seqrepo:/usr/local/share/seqrepo:ro biocommons/seqrepo-rest-service:0.2.2 seqrepo-rest-service /usr/local/share/seqrepo/2024-12-20`.
+When a change needs new data in both, re-record the hgvs cache first, since cassette
+recording reads hgvs data from it:
+
+```shell
+# hgvs cache: needs UTA and seqrepo-rest-service; does not touch cassettes
+export UTA_DB_URL=postgresql://anonymous@localhost:5432/uta/uta_20241220
+make record-hgvs-cache
+
+# VCR cassettes: needs seqrepo-rest-service
+make clean-cassettes
+pytest --vcr-record=once
+```
+
+`SEQREPO_REST_URL` (default `http://localhost:5000/seqrepo`) sets the REST service used
+by both.
+
 ## Running the Notebooks
 
 The notebooks **do not** require you to setup SeqRepo or UTA from

tests/conftest.py

@@ -1,10 +1,68 @@
 import os
+from collections.abc import Iterator
+from pathlib import Path
 
+import hgvs.dataproviders.uta
 import pytest
 from biocommons.seqrepo import SeqRepo
+from hgvs_cache_json import json_to_pickle
 
 from ga4gh.vrs.dataproxy import SeqRepoDataProxy, SeqRepoRESTDataProxy
 
+HGVS_CACHE_MODES = ("learn", "run", "verify")
+HGVS_CACHE_JSON = Path(__file__).parent / "data" / "hgvs_cache.json"
+
+
+@pytest.fixture(scope="session", autouse=True)
+def hgvs_cached_data_provider(
+    tmp_path_factory: pytest.TempPathFactory,
+) -> Iterator[hgvs.dataproviders.uta.UTABase | None]:
+    """Serve hgvs data provider lookups (UTA queries and hgvs sequence fetches) from a
+    recorded cache, so tests don't need a UTA database or network access.
+
+    Every `HgvsTools` gets this one data provider, so in learn mode they cannot
+    overwrite each other's cache entries. `VRS_HGVS_CACHE_MODE` is `run` (default),
+    `learn`, or `verify`, or empty to query UTA directly without the cache. In run
+    mode, a lookup missing from the cache raises `HGVSDataNotAvailableError`.
+
+    hgvs keeps its cache as a pickle, so in run and verify modes the committed
+    `tests/data/hgvs_cache.json` is converted to a temporary pickle for hgvs to load
+    (see `hgvs_cache_json.py`). Learn mode records a new cache, starting empty, into
+    the pickle file named by `VRS_HGVS_CACHE_FILE`. To re-record, run
+    `make record-hgvs-cache` with UTA_DB_URL pointing at a UTA instance and a
+    seqrepo-rest-service running at SEQREPO_REST_URL; it converts the recorded pickle
+    to `tests/data/hgvs_cache.json`.
+
+    Learn and verify modes, and an empty mode, query UTA and fetch sequences live. The
+    VCR cassettes don't contain those hgvs sequence requests (to HGVS_SEQREPO_URL), so
+    run these modes with `pytest --disable-vcr`.
+    """
+    mode = os.environ.get("VRS_HGVS_CACHE_MODE", "run")
+    if not mode:
+        yield None
+        return
+    if mode not in HGVS_CACHE_MODES:
+        msg = f"VRS_HGVS_CACHE_MODE must be one of {HGVS_CACHE_MODES} or empty, got {mode!r}"
+        raise pytest.UsageError(msg)
+    if mode == "learn":
+        cache_file = os.environ.get("VRS_HGVS_CACHE_FILE")
+        if not cache_file:
+            msg = "VRS_HGVS_CACHE_FILE must name the pickle file to record into in learn mode"
+            raise pytest.UsageError(msg)
+    else:
+        cache_file = str(tmp_path_factory.mktemp("hgvs_cache") / "hgvs_cache.pkl")
+        json_to_pickle(HGVS_CACHE_JSON, Path(cache_file))
+
+    provider = hgvs.dataproviders.uta.connect(mode=mode, cache=cache_file)
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setattr(
+            hgvs.dataproviders.uta, "connect", lambda *_args, **_kwargs: provider
+        )
+        # HgvsTools.close() closes its data provider, but this one is shared
+        mp.setattr(provider, "close", lambda: None)
+        yield provider
+    provider.close()
+
 
 def remove_request_headers(request):
     """Remove all headers from VCR request before recording."""