From eb540b736ee9f38340a9ad32e775bf6237515c65 Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Thu, 22 Oct 2026 15:57:19 +0000 Subject: [PATCH 1/2] Cap container memory+swap together and allow Postgres server settings A MEM_LIMIT now also sets the memory+swap limit to the same value so the test container cannot spill into swap on small CI agents (override with PGDEVKIT_TESTDB_MEMSWAP_LIMIT, -1 = unlimited). PGDEVKIT_TESTDB_PG_SETTINGS adds -c key=value server settings, e.g. shared_buffers=128MB max_connections=50. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 9 ++++++++- pgdevkit/testdb/_docker.py | 8 +++++++- pgdevkit/testdb/constants.py | 19 ++++++++++++++++++- pgdevkit/testdb/container.py | 1 + pgdevkit/testdb/mssql/constants.py | 1 + pgdevkit/testdb/mssql/container.py | 1 + tests/testdb/test_constants.py | 17 +++++++++++++++++ tests/testdb/test_container.py | 1 + tests/testdb/test_docker.py | 18 +++++++++++++++++- 9 files changed, 71 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 86e4d2c..be69871 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,12 @@ for parallel query workers), `PGDEVKIT_TESTDB_MEM_LIMIT` (e.g. `1g`), `PGDEVKIT_TESTDB_CPUS` (e.g. `2` or `0.5`), and `PGDEVKIT_TESTDB_TMPFS` — docker-CLI `--tmpfs` syntax (`/path:options`), semicolon-separated for multiple mounts, e.g. `/var/lib/postgresql/data:size=512m`. +Setting `PGDEVKIT_TESTDB_MEM_LIMIT` also sets the container's memory+swap +limit to the same value (`--memory-swap=`, so it can't spill into swap); +override with `PGDEVKIT_TESTDB_MEMSWAP_LIMIT` (`-1` = unlimited swap). +`PGDEVKIT_TESTDB_PG_SETTINGS` adds server settings as whitespace/comma-separated +`key=value` pairs, passed as `-c key=value`, e.g. +`shared_buffers=128MB max_connections=50` for small CI hosts. All of the above (including `_HOST`/`_PORT`/`_USER`/`_PASSWORD`/`_IMAGE`) only take effect when pgdevkit actually creates the container, not when it @@ -295,7 +301,8 @@ The same Docker-level resource knobs as the Postgres container are available here too: `PGDEVKIT_TESTDB_MSSQL_SHM_SIZE`, `PGDEVKIT_TESTDB_MSSQL_MEM_LIMIT` (a container-level cgroup limit, distinct from `_MEMORY_LIMIT_MB` above which only tunes SQL Server's own internal -memory management), `PGDEVKIT_TESTDB_MSSQL_CPUS`, and +memory management; like the Postgres one it also caps memory+swap, override via +`PGDEVKIT_TESTDB_MSSQL_MEMSWAP_LIMIT`), `PGDEVKIT_TESTDB_MSSQL_CPUS`, and `PGDEVKIT_TESTDB_MSSQL_TMPFS`. As with the Postgres container, these only take effect when pgdevkit creates the container, not when restarting an existing stopped one (`docker rm -f pgdevkit-mssql` first if you've changed diff --git a/pgdevkit/testdb/_docker.py b/pgdevkit/testdb/_docker.py index 9a1bebd..48cc39d 100644 --- a/pgdevkit/testdb/_docker.py +++ b/pgdevkit/testdb/_docker.py @@ -57,9 +57,12 @@ def _parse_tmpfs(spec: str) -> dict[str, str]: return mounts -def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str) -> dict: +def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str, memswap_limit: str = "") -> dict: """Build the `containers.run()` kwargs for the tmpfs/shm_size/mem_limit/cpus overrides, omitting any that aren't set so Docker's own defaults apply. + A mem_limit without an explicit memswap_limit also caps memory+swap at the + same value (the `--memory-swap=` idiom), so the container can't + spill past its limit into swap; pass memswap_limit="-1" to allow unlimited swap. Shared by the Postgres and MSSQL container modules -- these are Docker-level resource knobs, not anything specific to either engine.""" kwargs: dict = {} @@ -69,6 +72,9 @@ def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str) -> kwargs["shm_size"] = shm_size if mem_limit: kwargs["mem_limit"] = mem_limit + kwargs["memswap_limit"] = memswap_limit or mem_limit + elif memswap_limit: + kwargs["memswap_limit"] = memswap_limit if cpus: kwargs["nano_cpus"] = int(float(cpus) * 1_000_000_000) return kwargs diff --git a/pgdevkit/testdb/constants.py b/pgdevkit/testdb/constants.py index 9916041..ac1581c 100644 --- a/pgdevkit/testdb/constants.py +++ b/pgdevkit/testdb/constants.py @@ -16,6 +16,8 @@ TMPFS = os.environ.get("PGDEVKIT_TESTDB_TMPFS", "") SHM_SIZE = os.environ.get("PGDEVKIT_TESTDB_SHM_SIZE", "") MEM_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MEM_LIMIT", "") +# Defaults to MEM_LIMIT (no swap) when that is set; "-1" allows unlimited swap. +MEMSWAP_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MEMSWAP_LIMIT", "") CPUS = os.environ.get("PGDEVKIT_TESTDB_CPUS", "") PG_SPEED_FLAGS = ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] # Production connects with session timezone=UTC; the container image's own default @@ -24,7 +26,22 @@ # rows near local midnight. Forcing UTC here -- both as the server's own timezone GUC # and as PGTZ/TZ for any client library that consults the environment instead -- keeps # test parity with prod instead of depending on the host/image's locale. -PG_STARTUP_FLAGS = [*PG_SPEED_FLAGS, "-c", "timezone=UTC"] +# Extra server settings as whitespace/comma-separated key=value pairs (e.g. +# "shared_buffers=128MB max_connections=50"), each passed as `-c key=value` after the +# defaults so it wins -- for capping a test server's memory footprint on small CI hosts. +PG_SETTINGS = os.environ.get("PGDEVKIT_TESTDB_PG_SETTINGS", "") + + +def pg_settings_flags(spec: str) -> list[str]: + flags: list[str] = [] + for pair in spec.replace(",", " ").split(): + if "=" not in pair: + raise ValueError(f"PGDEVKIT_TESTDB_PG_SETTINGS entry {pair!r} is not key=value") + flags += ["-c", pair] + return flags + + +PG_STARTUP_FLAGS = [*PG_SPEED_FLAGS, "-c", "timezone=UTC", *pg_settings_flags(PG_SETTINGS)] def conninfo(dbname: str, *, connect_timeout: int | None = None) -> str: diff --git a/pgdevkit/testdb/container.py b/pgdevkit/testdb/container.py index ac8081b..64c2b21 100644 --- a/pgdevkit/testdb/container.py +++ b/pgdevkit/testdb/container.py @@ -48,6 +48,7 @@ def _create_container(client: docker.DockerClient) -> None: shm_size=constants.SHM_SIZE, mem_limit=constants.MEM_LIMIT, cpus=constants.CPUS, + memswap_limit=constants.MEMSWAP_LIMIT, ), ) except docker.errors.APIError as e: diff --git a/pgdevkit/testdb/mssql/constants.py b/pgdevkit/testdb/mssql/constants.py index 4dce5e2..98df12f 100644 --- a/pgdevkit/testdb/mssql/constants.py +++ b/pgdevkit/testdb/mssql/constants.py @@ -30,6 +30,7 @@ TMPFS = os.environ.get("PGDEVKIT_TESTDB_MSSQL_TMPFS", "") SHM_SIZE = os.environ.get("PGDEVKIT_TESTDB_MSSQL_SHM_SIZE", "") MEM_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MSSQL_MEM_LIMIT", "") +MEMSWAP_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MSSQL_MEMSWAP_LIMIT", "") CPUS = os.environ.get("PGDEVKIT_TESTDB_MSSQL_CPUS", "") diff --git a/pgdevkit/testdb/mssql/container.py b/pgdevkit/testdb/mssql/container.py index fb0d953..4c01b5b 100644 --- a/pgdevkit/testdb/mssql/container.py +++ b/pgdevkit/testdb/mssql/container.py @@ -45,6 +45,7 @@ def _create_container(client: docker.DockerClient) -> None: shm_size=constants.SHM_SIZE, mem_limit=constants.MEM_LIMIT, cpus=constants.CPUS, + memswap_limit=constants.MEMSWAP_LIMIT, ), ) except docker.errors.APIError as e: diff --git a/tests/testdb/test_constants.py b/tests/testdb/test_constants.py index a790ddb..d650100 100644 --- a/tests/testdb/test_constants.py +++ b/tests/testdb/test_constants.py @@ -2,6 +2,8 @@ import importlib +import pytest + from pgdevkit.testdb import constants @@ -65,3 +67,18 @@ def test_env_vars_override_docker_resource_defaults(monkeypatch): finally: monkeypatch.undo() importlib.reload(constants) + + +def test_pg_settings_flags_become_dash_c_pairs(): + assert constants.pg_settings_flags("shared_buffers=128MB, max_connections=50") == [ + "-c", "shared_buffers=128MB", "-c", "max_connections=50", + ] + + +def test_pg_settings_flags_empty_spec_adds_nothing(): + assert constants.pg_settings_flags("") == [] + + +def test_pg_settings_flags_rejects_entry_without_equals(): + with pytest.raises(ValueError, match="key=value"): + constants.pg_settings_flags("shared_buffers") diff --git a/tests/testdb/test_container.py b/tests/testdb/test_container.py index 4177a5e..8905fbc 100644 --- a/tests/testdb/test_container.py +++ b/tests/testdb/test_container.py @@ -100,6 +100,7 @@ def test_create_container_passes_through_resource_overrides(monkeypatch): assert client.containers.run_kwargs["tmpfs"] == {"/var/lib/postgresql/data": "size=512m"} assert client.containers.run_kwargs["shm_size"] == "256m" assert client.containers.run_kwargs["mem_limit"] == "1g" + assert client.containers.run_kwargs["memswap_limit"] == "1g" assert client.containers.run_kwargs["nano_cpus"] == 2_000_000_000 diff --git a/tests/testdb/test_docker.py b/tests/testdb/test_docker.py index 26e99ce..d6df7a9 100644 --- a/tests/testdb/test_docker.py +++ b/tests/testdb/test_docker.py @@ -33,7 +33,23 @@ def test_resource_kwargs_all_unset_returns_empty_dict(): def test_resource_kwargs_only_includes_set_values(): kwargs = resource_kwargs(tmpfs="/data:size=512m", shm_size="", mem_limit="1g", cpus="") - assert kwargs == {"tmpfs": {"/data": "size=512m"}, "mem_limit": "1g"} + assert kwargs == {"tmpfs": {"/data": "size=512m"}, "mem_limit": "1g", "memswap_limit": "1g"} + + +def test_resource_kwargs_memswap_limit_defaults_to_mem_limit(): + assert resource_kwargs(tmpfs="", shm_size="", mem_limit="512m", cpus="") == { + "mem_limit": "512m", + "memswap_limit": "512m", + } + + +def test_resource_kwargs_explicit_memswap_limit_wins(): + kwargs = resource_kwargs(tmpfs="", shm_size="", mem_limit="512m", cpus="", memswap_limit="1g") + assert kwargs == {"mem_limit": "512m", "memswap_limit": "1g"} + + +def test_resource_kwargs_memswap_limit_without_mem_limit_is_passed_through(): + assert resource_kwargs(tmpfs="", shm_size="", mem_limit="", cpus="", memswap_limit="-1") == {"memswap_limit": "-1"} def test_resource_kwargs_cpus_converted_to_nano_cpus(): From 621bfe092e2592e63e714418d75daffbc8c81f6a Mon Sep 17 00:00:00 2001 From: Adrian Ehrsam Date: Thu, 22 Oct 2026 16:07:40 +0000 Subject: [PATCH 2/2] Review fixes: whitespace-only PG settings split, ignore swap limit without memory limit; bump to 0.17.0 Co-Authored-By: Claude Sonnet 5.5 --- README.md | 5 +++-- pgdevkit/testdb/_docker.py | 3 +-- pgdevkit/testdb/constants.py | 4 ++-- pyproject.toml | 2 +- tests/testdb/test_constants.py | 2 +- tests/testdb/test_docker.py | 4 ++-- uv.lock | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index be69871..857400a 100644 --- a/README.md +++ b/README.md @@ -261,8 +261,9 @@ docker-CLI `--tmpfs` syntax (`/path:options`), semicolon-separated for multiple mounts, e.g. `/var/lib/postgresql/data:size=512m`. Setting `PGDEVKIT_TESTDB_MEM_LIMIT` also sets the container's memory+swap limit to the same value (`--memory-swap=`, so it can't spill into swap); -override with `PGDEVKIT_TESTDB_MEMSWAP_LIMIT` (`-1` = unlimited swap). -`PGDEVKIT_TESTDB_PG_SETTINGS` adds server settings as whitespace/comma-separated +override with `PGDEVKIT_TESTDB_MEMSWAP_LIMIT` (`-1` = unlimited swap; ignored +without a `MEM_LIMIT`). +`PGDEVKIT_TESTDB_PG_SETTINGS` adds server settings as whitespace-separated `key=value` pairs, passed as `-c key=value`, e.g. `shared_buffers=128MB max_connections=50` for small CI hosts. diff --git a/pgdevkit/testdb/_docker.py b/pgdevkit/testdb/_docker.py index 48cc39d..c3a49b0 100644 --- a/pgdevkit/testdb/_docker.py +++ b/pgdevkit/testdb/_docker.py @@ -63,6 +63,7 @@ def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str, mem A mem_limit without an explicit memswap_limit also caps memory+swap at the same value (the `--memory-swap=` idiom), so the container can't spill past its limit into swap; pass memswap_limit="-1" to allow unlimited swap. + A memswap_limit without a mem_limit is ignored (Docker/Podman need both). Shared by the Postgres and MSSQL container modules -- these are Docker-level resource knobs, not anything specific to either engine.""" kwargs: dict = {} @@ -73,8 +74,6 @@ def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str, mem if mem_limit: kwargs["mem_limit"] = mem_limit kwargs["memswap_limit"] = memswap_limit or mem_limit - elif memswap_limit: - kwargs["memswap_limit"] = memswap_limit if cpus: kwargs["nano_cpus"] = int(float(cpus) * 1_000_000_000) return kwargs diff --git a/pgdevkit/testdb/constants.py b/pgdevkit/testdb/constants.py index ac1581c..2acfb7c 100644 --- a/pgdevkit/testdb/constants.py +++ b/pgdevkit/testdb/constants.py @@ -26,7 +26,7 @@ # rows near local midnight. Forcing UTC here -- both as the server's own timezone GUC # and as PGTZ/TZ for any client library that consults the environment instead -- keeps # test parity with prod instead of depending on the host/image's locale. -# Extra server settings as whitespace/comma-separated key=value pairs (e.g. +# Extra server settings as whitespace-separated key=value pairs (no spaces in values; e.g. # "shared_buffers=128MB max_connections=50"), each passed as `-c key=value` after the # defaults so it wins -- for capping a test server's memory footprint on small CI hosts. PG_SETTINGS = os.environ.get("PGDEVKIT_TESTDB_PG_SETTINGS", "") @@ -34,7 +34,7 @@ def pg_settings_flags(spec: str) -> list[str]: flags: list[str] = [] - for pair in spec.replace(",", " ").split(): + for pair in spec.split(): if "=" not in pair: raise ValueError(f"PGDEVKIT_TESTDB_PG_SETTINGS entry {pair!r} is not key=value") flags += ["-c", pair] diff --git a/pyproject.toml b/pyproject.toml index 44e127c..8414a42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ packages = ["pgdevkit"] [project] name = "pgdevkit" -version = "0.16.1" +version = "0.17.0" description = "A helper for developing with Postgres" readme = "README.md" requires-python = ">=3.14" diff --git a/tests/testdb/test_constants.py b/tests/testdb/test_constants.py index d650100..5fedcc7 100644 --- a/tests/testdb/test_constants.py +++ b/tests/testdb/test_constants.py @@ -70,7 +70,7 @@ def test_env_vars_override_docker_resource_defaults(monkeypatch): def test_pg_settings_flags_become_dash_c_pairs(): - assert constants.pg_settings_flags("shared_buffers=128MB, max_connections=50") == [ + assert constants.pg_settings_flags("shared_buffers=128MB max_connections=50") == [ "-c", "shared_buffers=128MB", "-c", "max_connections=50", ] diff --git a/tests/testdb/test_docker.py b/tests/testdb/test_docker.py index d6df7a9..a465c9f 100644 --- a/tests/testdb/test_docker.py +++ b/tests/testdb/test_docker.py @@ -48,8 +48,8 @@ def test_resource_kwargs_explicit_memswap_limit_wins(): assert kwargs == {"mem_limit": "512m", "memswap_limit": "1g"} -def test_resource_kwargs_memswap_limit_without_mem_limit_is_passed_through(): - assert resource_kwargs(tmpfs="", shm_size="", mem_limit="", cpus="", memswap_limit="-1") == {"memswap_limit": "-1"} +def test_resource_kwargs_memswap_limit_without_mem_limit_is_ignored(): + assert resource_kwargs(tmpfs="", shm_size="", mem_limit="", cpus="", memswap_limit="1g") == {} def test_resource_kwargs_cpus_converted_to_nano_cpus(): diff --git a/uv.lock b/uv.lock index efe15c5..ef76b18 100644 --- a/uv.lock +++ b/uv.lock @@ -451,7 +451,7 @@ wheels = [ [[package]] name = "pgdevkit" -version = "0.16.1" +version = "0.17.0" source = { editable = "." } dependencies = [ { name = "docker" },