diff --git a/README.md b/README.md index 86e4d2c..857400a 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,13 @@ for parallel query workers), `PGDEVKIT_TESTDB_MEM_LIMIT` (e.g. `1g`), `PGDEVKIT_TESTDB_CPUS` (e.g. `2` or `0.5`), and `PGDEVKIT_TESTDB_TMPFS` — docker-CLI `--tmpfs` syntax (`/path:options`), semicolon-separated for multiple mounts, e.g. `/var/lib/postgresql/data:size=512m`. +Setting `PGDEVKIT_TESTDB_MEM_LIMIT` also sets the container's memory+swap +limit to the same value (`--memory-swap=`, so it can't spill into swap); +override with `PGDEVKIT_TESTDB_MEMSWAP_LIMIT` (`-1` = unlimited swap; ignored +without a `MEM_LIMIT`). +`PGDEVKIT_TESTDB_PG_SETTINGS` adds server settings as whitespace-separated +`key=value` pairs, passed as `-c key=value`, e.g. +`shared_buffers=128MB max_connections=50` for small CI hosts. All of the above (including `_HOST`/`_PORT`/`_USER`/`_PASSWORD`/`_IMAGE`) only take effect when pgdevkit actually creates the container, not when it @@ -295,7 +302,8 @@ The same Docker-level resource knobs as the Postgres container are available here too: `PGDEVKIT_TESTDB_MSSQL_SHM_SIZE`, `PGDEVKIT_TESTDB_MSSQL_MEM_LIMIT` (a container-level cgroup limit, distinct from `_MEMORY_LIMIT_MB` above which only tunes SQL Server's own internal -memory management), `PGDEVKIT_TESTDB_MSSQL_CPUS`, and +memory management; like the Postgres one it also caps memory+swap, override via +`PGDEVKIT_TESTDB_MSSQL_MEMSWAP_LIMIT`), `PGDEVKIT_TESTDB_MSSQL_CPUS`, and `PGDEVKIT_TESTDB_MSSQL_TMPFS`. As with the Postgres container, these only take effect when pgdevkit creates the container, not when restarting an existing stopped one (`docker rm -f pgdevkit-mssql` first if you've changed diff --git a/pgdevkit/testdb/_docker.py b/pgdevkit/testdb/_docker.py index 9a1bebd..c3a49b0 100644 --- a/pgdevkit/testdb/_docker.py +++ b/pgdevkit/testdb/_docker.py @@ -57,9 +57,13 @@ def _parse_tmpfs(spec: str) -> dict[str, str]: return mounts -def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str) -> dict: +def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str, memswap_limit: str = "") -> dict: """Build the `containers.run()` kwargs for the tmpfs/shm_size/mem_limit/cpus overrides, omitting any that aren't set so Docker's own defaults apply. + A mem_limit without an explicit memswap_limit also caps memory+swap at the + same value (the `--memory-swap=` idiom), so the container can't + spill past its limit into swap; pass memswap_limit="-1" to allow unlimited swap. + A memswap_limit without a mem_limit is ignored (Docker/Podman need both). Shared by the Postgres and MSSQL container modules -- these are Docker-level resource knobs, not anything specific to either engine.""" kwargs: dict = {} @@ -69,6 +73,7 @@ def resource_kwargs(*, tmpfs: str, shm_size: str, mem_limit: str, cpus: str) -> kwargs["shm_size"] = shm_size if mem_limit: kwargs["mem_limit"] = mem_limit + kwargs["memswap_limit"] = memswap_limit or mem_limit if cpus: kwargs["nano_cpus"] = int(float(cpus) * 1_000_000_000) return kwargs diff --git a/pgdevkit/testdb/constants.py b/pgdevkit/testdb/constants.py index 9916041..2acfb7c 100644 --- a/pgdevkit/testdb/constants.py +++ b/pgdevkit/testdb/constants.py @@ -16,6 +16,8 @@ TMPFS = os.environ.get("PGDEVKIT_TESTDB_TMPFS", "") SHM_SIZE = os.environ.get("PGDEVKIT_TESTDB_SHM_SIZE", "") MEM_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MEM_LIMIT", "") +# Defaults to MEM_LIMIT (no swap) when that is set; "-1" allows unlimited swap. +MEMSWAP_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MEMSWAP_LIMIT", "") CPUS = os.environ.get("PGDEVKIT_TESTDB_CPUS", "") PG_SPEED_FLAGS = ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] # Production connects with session timezone=UTC; the container image's own default @@ -24,7 +26,22 @@ # rows near local midnight. Forcing UTC here -- both as the server's own timezone GUC # and as PGTZ/TZ for any client library that consults the environment instead -- keeps # test parity with prod instead of depending on the host/image's locale. -PG_STARTUP_FLAGS = [*PG_SPEED_FLAGS, "-c", "timezone=UTC"] +# Extra server settings as whitespace-separated key=value pairs (no spaces in values; e.g. +# "shared_buffers=128MB max_connections=50"), each passed as `-c key=value` after the +# defaults so it wins -- for capping a test server's memory footprint on small CI hosts. +PG_SETTINGS = os.environ.get("PGDEVKIT_TESTDB_PG_SETTINGS", "") + + +def pg_settings_flags(spec: str) -> list[str]: + flags: list[str] = [] + for pair in spec.split(): + if "=" not in pair: + raise ValueError(f"PGDEVKIT_TESTDB_PG_SETTINGS entry {pair!r} is not key=value") + flags += ["-c", pair] + return flags + + +PG_STARTUP_FLAGS = [*PG_SPEED_FLAGS, "-c", "timezone=UTC", *pg_settings_flags(PG_SETTINGS)] def conninfo(dbname: str, *, connect_timeout: int | None = None) -> str: diff --git a/pgdevkit/testdb/container.py b/pgdevkit/testdb/container.py index ac8081b..64c2b21 100644 --- a/pgdevkit/testdb/container.py +++ b/pgdevkit/testdb/container.py @@ -48,6 +48,7 @@ def _create_container(client: docker.DockerClient) -> None: shm_size=constants.SHM_SIZE, mem_limit=constants.MEM_LIMIT, cpus=constants.CPUS, + memswap_limit=constants.MEMSWAP_LIMIT, ), ) except docker.errors.APIError as e: diff --git a/pgdevkit/testdb/mssql/constants.py b/pgdevkit/testdb/mssql/constants.py index 4dce5e2..98df12f 100644 --- a/pgdevkit/testdb/mssql/constants.py +++ b/pgdevkit/testdb/mssql/constants.py @@ -30,6 +30,7 @@ TMPFS = os.environ.get("PGDEVKIT_TESTDB_MSSQL_TMPFS", "") SHM_SIZE = os.environ.get("PGDEVKIT_TESTDB_MSSQL_SHM_SIZE", "") MEM_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MSSQL_MEM_LIMIT", "") +MEMSWAP_LIMIT = os.environ.get("PGDEVKIT_TESTDB_MSSQL_MEMSWAP_LIMIT", "") CPUS = os.environ.get("PGDEVKIT_TESTDB_MSSQL_CPUS", "") diff --git a/pgdevkit/testdb/mssql/container.py b/pgdevkit/testdb/mssql/container.py index fb0d953..4c01b5b 100644 --- a/pgdevkit/testdb/mssql/container.py +++ b/pgdevkit/testdb/mssql/container.py @@ -45,6 +45,7 @@ def _create_container(client: docker.DockerClient) -> None: shm_size=constants.SHM_SIZE, mem_limit=constants.MEM_LIMIT, cpus=constants.CPUS, + memswap_limit=constants.MEMSWAP_LIMIT, ), ) except docker.errors.APIError as e: diff --git a/pyproject.toml b/pyproject.toml index 44e127c..8414a42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ packages = ["pgdevkit"] [project] name = "pgdevkit" -version = "0.16.1" +version = "0.17.0" description = "A helper for developing with Postgres" readme = "README.md" requires-python = ">=3.14" diff --git a/tests/testdb/test_constants.py b/tests/testdb/test_constants.py index a790ddb..5fedcc7 100644 --- a/tests/testdb/test_constants.py +++ b/tests/testdb/test_constants.py @@ -2,6 +2,8 @@ import importlib +import pytest + from pgdevkit.testdb import constants @@ -65,3 +67,18 @@ def test_env_vars_override_docker_resource_defaults(monkeypatch): finally: monkeypatch.undo() importlib.reload(constants) + + +def test_pg_settings_flags_become_dash_c_pairs(): + assert constants.pg_settings_flags("shared_buffers=128MB max_connections=50") == [ + "-c", "shared_buffers=128MB", "-c", "max_connections=50", + ] + + +def test_pg_settings_flags_empty_spec_adds_nothing(): + assert constants.pg_settings_flags("") == [] + + +def test_pg_settings_flags_rejects_entry_without_equals(): + with pytest.raises(ValueError, match="key=value"): + constants.pg_settings_flags("shared_buffers") diff --git a/tests/testdb/test_container.py b/tests/testdb/test_container.py index 4177a5e..8905fbc 100644 --- a/tests/testdb/test_container.py +++ b/tests/testdb/test_container.py @@ -100,6 +100,7 @@ def test_create_container_passes_through_resource_overrides(monkeypatch): assert client.containers.run_kwargs["tmpfs"] == {"/var/lib/postgresql/data": "size=512m"} assert client.containers.run_kwargs["shm_size"] == "256m" assert client.containers.run_kwargs["mem_limit"] == "1g" + assert client.containers.run_kwargs["memswap_limit"] == "1g" assert client.containers.run_kwargs["nano_cpus"] == 2_000_000_000 diff --git a/tests/testdb/test_docker.py b/tests/testdb/test_docker.py index 26e99ce..a465c9f 100644 --- a/tests/testdb/test_docker.py +++ b/tests/testdb/test_docker.py @@ -33,7 +33,23 @@ def test_resource_kwargs_all_unset_returns_empty_dict(): def test_resource_kwargs_only_includes_set_values(): kwargs = resource_kwargs(tmpfs="/data:size=512m", shm_size="", mem_limit="1g", cpus="") - assert kwargs == {"tmpfs": {"/data": "size=512m"}, "mem_limit": "1g"} + assert kwargs == {"tmpfs": {"/data": "size=512m"}, "mem_limit": "1g", "memswap_limit": "1g"} + + +def test_resource_kwargs_memswap_limit_defaults_to_mem_limit(): + assert resource_kwargs(tmpfs="", shm_size="", mem_limit="512m", cpus="") == { + "mem_limit": "512m", + "memswap_limit": "512m", + } + + +def test_resource_kwargs_explicit_memswap_limit_wins(): + kwargs = resource_kwargs(tmpfs="", shm_size="", mem_limit="512m", cpus="", memswap_limit="1g") + assert kwargs == {"mem_limit": "512m", "memswap_limit": "1g"} + + +def test_resource_kwargs_memswap_limit_without_mem_limit_is_ignored(): + assert resource_kwargs(tmpfs="", shm_size="", mem_limit="", cpus="", memswap_limit="1g") == {} def test_resource_kwargs_cpus_converted_to_nano_cpus(): diff --git a/uv.lock b/uv.lock index efe15c5..ef76b18 100644 --- a/uv.lock +++ b/uv.lock @@ -451,7 +451,7 @@ wheels = [ [[package]] name = "pgdevkit" -version = "0.16.1" +version = "0.17.0" source = { editable = "." } dependencies = [ { name = "docker" },