From 25c807e1bf9af4ff4b98f7edbb8ef8a60f6be43d Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 16:48:47 +0100 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20decode=20half-yearly=20CBS=20period?= =?UTF-8?q?=20codes=20(2023HJ01=20=E2=86=92=202023=20H1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- src/cbspy/_periods.py | 5 +++++ tests/test_periods.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/cbspy/_periods.py b/src/cbspy/_periods.py index ac01222..92a2f0e 100644 --- a/src/cbspy/_periods.py +++ b/src/cbspy/_periods.py @@ -16,6 +16,7 @@ ] _YEARLY = re.compile(r"^(\d{4})JJ00$") +_HALFYEARLY = re.compile(r"^(\d{4})HJ0([12])$") _QUARTERLY = re.compile(r"^(\d{4})KW0([1-4])$") _MONTHLY = re.compile(r"^(\d{4})MM(\d{2})$") @@ -25,6 +26,7 @@ def decode_period(raw: str) -> str: Examples: 2023JJ00 -> "2023" + 2023HJ01 -> "2023 H1" 2023KW01 -> "2023 Q1" 2023MM03 -> "2023 March" """ @@ -33,6 +35,9 @@ def decode_period(raw: str) -> str: if m := _YEARLY.match(s): return m.group(1) + if m := _HALFYEARLY.match(s): + return f"{m.group(1)} H{m.group(2)}" + if m := _QUARTERLY.match(s): return f"{m.group(1)} Q{m.group(2)}" diff --git a/tests/test_periods.py b/tests/test_periods.py index 67c5b59..c326103 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -8,6 +8,9 @@ [ ("2023JJ00", "2023"), ("1950JJ00", "1950"), + ("2023HJ01", "2023 H1"), + ("2023HJ02", "2023 H2"), + ("1990HJ01", "1990 H1"), ("2023KW01", "2023 Q1"), ("2023KW02", "2023 Q2"), ("2023KW03", "2023 Q3"), From 1a19d83cb6cd4546a23aa402a387b12e5eab4738 Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 16:49:25 +0100 Subject: [PATCH 2/7] perf: switch to ODataFeed endpoint for larger page sizes ODataFeed has no 10,000-row page limit, reducing HTTP round-trips for large tables from 50+ requests to 1-5. Co-Authored-By: Claude Opus 4.6 --- src/cbspy/_odata.py | 2 +- tests/test_client.py | 18 +++++++++--------- tests/test_odata.py | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/cbspy/_odata.py b/src/cbspy/_odata.py index 9f755c9..cd862e6 100644 --- a/src/cbspy/_odata.py +++ b/src/cbspy/_odata.py @@ -7,7 +7,7 @@ from cbspy.exceptions import APIError, TableNotFoundError -_ODATA_API = "/ODataApi/odata" +_ODATA_API = "/ODataFeed/OData" _CATALOG = "/ODataCatalog/Tables" _MAX_RETRIES = 1 diff --git a/tests/test_client.py b/tests/test_client.py index c315bf2..20263bd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -79,7 +79,7 @@ def test_filters_by_language(self): class TestGetMetadata: @respx.mock def test_returns_table_metadata(self): - respx.get(f"{BASE}/ODataApi/odata/37296eng/TableInfos").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/TableInfos").mock( return_value=httpx.Response( 200, json={ @@ -95,7 +95,7 @@ def test_returns_table_metadata(self): }, ) ) - respx.get(f"{BASE}/ODataApi/odata/37296eng/DataProperties").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( return_value=httpx.Response( 200, json={ @@ -134,7 +134,7 @@ def test_returns_table_metadata(self): @respx.mock def test_not_found_raises(self): - respx.get(f"{BASE}/ODataApi/odata/FAKE/TableInfos").mock(return_value=httpx.Response(404, text="Not found")) + respx.get(f"{BASE}/ODataFeed/OData/FAKE/TableInfos").mock(return_value=httpx.Response(404, text="Not found")) client = Client() with pytest.raises(TableNotFoundError): client.get_metadata("FAKE") @@ -143,7 +143,7 @@ def test_not_found_raises(self): class TestGetData: @respx.mock def test_returns_dataframe_with_resolved_columns(self): - respx.get(f"{BASE}/ODataApi/odata/37296eng/DataProperties").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( return_value=httpx.Response( 200, json={ @@ -180,7 +180,7 @@ def test_returns_dataframe_with_resolved_columns(self): }, ) ) - respx.get(f"{BASE}/ODataApi/odata/37296eng/TypedDataSet").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( return_value=httpx.Response( 200, json={ @@ -202,7 +202,7 @@ def test_returns_dataframe_with_resolved_columns(self): @respx.mock def test_get_data_with_periods_filter(self): - respx.get(f"{BASE}/ODataApi/odata/37296eng/DataProperties").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( return_value=httpx.Response( 200, json={ @@ -229,7 +229,7 @@ def test_get_data_with_periods_filter(self): }, ) ) - respx.get(f"{BASE}/ODataApi/odata/37296eng/TypedDataSet").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( return_value=httpx.Response( 200, json={ @@ -245,7 +245,7 @@ def test_get_data_with_periods_filter(self): @respx.mock def test_get_data_empty_dataset(self): - respx.get(f"{BASE}/ODataApi/odata/37296eng/DataProperties").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( return_value=httpx.Response( 200, json={ @@ -262,7 +262,7 @@ def test_get_data_empty_dataset(self): }, ) ) - respx.get(f"{BASE}/ODataApi/odata/37296eng/TypedDataSet").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( return_value=httpx.Response(200, json={"value": []}) ) client = Client() diff --git a/tests/test_odata.py b/tests/test_odata.py index 8994755..8698741 100644 --- a/tests/test_odata.py +++ b/tests/test_odata.py @@ -11,7 +11,7 @@ class TestGetJson: @respx.mock def test_returns_value_array(self): - respx.get(f"{BASE}/ODataApi/odata/37296eng/TypedDataSet").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( return_value=httpx.Response(200, json={"value": [{"ID": 0, "Periods": "2023JJ00"}]}) ) client = ODataClient(base_url=BASE, http_client=httpx.Client()) @@ -23,7 +23,7 @@ def test_follows_pagination(self): 200, json={ "value": [{"ID": 0}], - "odata.nextLink": f"{BASE}/ODataApi/odata/37296eng/TypedDataSet?$skip=1", + "odata.nextLink": f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet?$skip=1", }, ) page2 = httpx.Response( @@ -40,14 +40,14 @@ def test_follows_pagination(self): @respx.mock def test_404_raises_table_not_found(self): - respx.get(f"{BASE}/ODataApi/odata/FAKE/TypedDataSet").mock(return_value=httpx.Response(404, text="Not found")) + respx.get(f"{BASE}/ODataFeed/OData/FAKE/TypedDataSet").mock(return_value=httpx.Response(404, text="Not found")) client = ODataClient(base_url=BASE, http_client=httpx.Client()) with pytest.raises(TableNotFoundError): client.get_json("FAKE", "TypedDataSet") @respx.mock def test_500_raises_api_error(self): - respx.get(f"{BASE}/ODataApi/odata/37296eng/TypedDataSet").mock( + respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( return_value=httpx.Response(500, text="Internal Server Error") ) client = ODataClient(base_url=BASE, http_client=httpx.Client()) From 2422fcac509cdea24e2927e753bcc071a2c13150 Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 16:55:39 +0100 Subject: [PATCH 3/7] feat: add generic dimension filtering to get_data() Accept a filters parameter as either a raw OData string or a dict mapping dimension keys to value lists. periods param still works and is combined with filters using AND. Co-Authored-By: Claude Opus 4.6 --- src/cbspy/client.py | 39 ++++++++++++++-- tests/test_client.py | 109 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/cbspy/client.py b/src/cbspy/client.py index 234ae53..b6c54fe 100644 --- a/src/cbspy/client.py +++ b/src/cbspy/client.py @@ -85,12 +85,20 @@ def get_metadata(self, table_id: str) -> TableMetadata: properties=columns, ) - def get_data(self, table_id: str, periods: list[str] | None = None) -> pl.DataFrame: + def get_data( + self, + table_id: str, + periods: list[str] | None = None, + filters: dict[str, list[str]] | str | None = None, + ) -> pl.DataFrame: """Fetch dataset as a Polars DataFrame with human-readable column names. Args: table_id: CBS table identifier (e.g. "37296eng"). periods: Optional list of CBS period codes to filter by. + filters: Optional dimension filters. Either a raw OData $filter string + or a dict mapping dimension keys to value lists, e.g. + ``{"RegioS": ["GM0363", "GM0599"]}``. Returns: Polars DataFrame with resolved column names and decoded periods. @@ -99,10 +107,10 @@ def get_data(self, table_id: str, periods: list[str] | None = None) -> pl.DataFr column_map = {p["Key"]: p.get("Title", p["Key"]) for p in prop_rows} period_keys = {p["Key"] for p in prop_rows if p.get("Type") == "TimeDimension"} + filter_str = self._build_filter(periods, filters, period_keys) params: dict[str, str] | None = None - if periods: - period_filter = " or ".join(f"Periods eq '{p}'" for p in periods) - params = {"$filter": period_filter} + if filter_str: + params = {"$filter": filter_str} data_rows = self._odata.get_json(table_id, "TypedDataSet", params=params) @@ -124,6 +132,29 @@ def get_data(self, table_id: str, periods: list[str] | None = None) -> pl.DataFr return pl.DataFrame(renamed_rows) + @staticmethod + def _build_filter( + periods: list[str] | None, + filters: dict[str, list[str]] | str | None, + period_keys: set[str], + ) -> str: + """Compile periods and filters into an OData $filter string.""" + parts: list[str] = [] + + if periods and period_keys: + period_key = next(iter(period_keys)) + clause = " or ".join(f"{period_key} eq '{p}'" for p in periods) + parts.append(f"({clause})") + + if isinstance(filters, str): + parts.append(filters) + elif isinstance(filters, dict): + for dim, values in filters.items(): + clause = " or ".join(f"{dim} eq '{v}'" for v in values) + parts.append(f"({clause})") + + return " and ".join(parts) + @staticmethod def _parse_column(prop: dict[str, Any]) -> Column: """Convert a raw DataProperties entry to a Column model.""" diff --git a/tests/test_client.py b/tests/test_client.py index 20263bd..6c817bd 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -271,6 +271,115 @@ def test_get_data_empty_dataset(self): assert df.shape[0] == 0 +class TestGetDataFilters: + """Tests for the generic filters parameter on get_data().""" + + def _mock_props(self): + """Mock DataProperties with a time dimension and a geo dimension.""" + respx.get(f"{BASE}/ODataFeed/OData/71450ned/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Perioden", "Title": "Periods", "Type": "TimeDimension"}, + {"Key": "RegioS", "Title": "Region", "Type": "GeoDimension"}, + { + "Key": "Bevolking_1", + "Title": "Population", + "Type": "Topic", + "Datatype": "Long", + "Unit": "number", + }, + ] + }, + ) + ) + + @respx.mock + def test_filters_dict_single_dimension(self): + self._mock_props() + route = respx.get(f"{BASE}/ODataFeed/OData/71450ned/TypedDataSet").mock( + return_value=httpx.Response( + 200, json={"value": [{"ID": 0, "Perioden": "2023JJ00", "RegioS": "GM0363 ", "Bevolking_1": 921402}]} + ) + ) + with Client() as client: + df = client.get_data("71450ned", filters={"RegioS": ["GM0363"]}) + assert df.shape[0] == 1 + sent_params = dict(route.calls[0].request.url.params) + assert "$filter" in sent_params + assert "RegioS" in sent_params["$filter"] + + @respx.mock + def test_filters_dict_multiple_dimensions(self): + self._mock_props() + route = respx.get(f"{BASE}/ODataFeed/OData/71450ned/TypedDataSet").mock( + return_value=httpx.Response( + 200, json={"value": [{"ID": 0, "Perioden": "2023JJ00", "RegioS": "GM0363 ", "Bevolking_1": 921402}]} + ) + ) + with Client() as client: + client.get_data("71450ned", filters={"RegioS": ["GM0363"], "Perioden": ["2023JJ00"]}) + sent_params = dict(route.calls[0].request.url.params) + filter_str = sent_params["$filter"] + assert "RegioS" in filter_str + assert "Perioden" in filter_str + + @respx.mock + def test_filters_raw_string(self): + self._mock_props() + route = respx.get(f"{BASE}/ODataFeed/OData/71450ned/TypedDataSet").mock( + return_value=httpx.Response( + 200, json={"value": [{"ID": 0, "Perioden": "2023JJ00", "RegioS": "GM0363 ", "Bevolking_1": 921402}]} + ) + ) + with Client() as client: + client.get_data("71450ned", filters="RegioS eq 'GM0363'") + sent_params = dict(route.calls[0].request.url.params) + assert sent_params["$filter"] == "RegioS eq 'GM0363'" + + @respx.mock + def test_periods_param_still_works(self): + self._mock_props() + route = respx.get(f"{BASE}/ODataFeed/OData/71450ned/TypedDataSet").mock( + return_value=httpx.Response( + 200, json={"value": [{"ID": 0, "Perioden": "2023JJ00", "RegioS": "GM0363 ", "Bevolking_1": 921402}]} + ) + ) + with Client() as client: + client.get_data("71450ned", periods=["2023JJ00"]) + sent_params = dict(route.calls[0].request.url.params) + assert "Perioden" in sent_params["$filter"] + + @respx.mock + def test_filters_and_periods_combined(self): + self._mock_props() + route = respx.get(f"{BASE}/ODataFeed/OData/71450ned/TypedDataSet").mock( + return_value=httpx.Response( + 200, json={"value": [{"ID": 0, "Perioden": "2023JJ00", "RegioS": "GM0363 ", "Bevolking_1": 921402}]} + ) + ) + with Client() as client: + client.get_data("71450ned", periods=["2023JJ00"], filters={"RegioS": ["GM0363"]}) + sent_params = dict(route.calls[0].request.url.params) + filter_str = sent_params["$filter"] + assert "RegioS" in filter_str + assert "Perioden" in filter_str + + @respx.mock + def test_no_filters(self): + self._mock_props() + route = respx.get(f"{BASE}/ODataFeed/OData/71450ned/TypedDataSet").mock( + return_value=httpx.Response( + 200, json={"value": [{"ID": 0, "Perioden": "2023JJ00", "RegioS": "GM0363 ", "Bevolking_1": 921402}]} + ) + ) + with Client() as client: + client.get_data("71450ned") + sent_params = dict(route.calls[0].request.url.params) + assert "$filter" not in sent_params + + class TestClientLifecycle: def test_close_closes_owned_http_client(self): client = Client() From 0dfa82102b99d87563ab9bf1b976381c2eaebabf Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 17:00:25 +0100 Subject: [PATCH 4/7] feat: add column selection ($select) to get_data() Accept column names as human-readable titles or CBS keys. Maps titles back to keys for the $select OData parameter. Co-Authored-By: Claude Opus 4.6 --- src/cbspy/client.py | 19 ++++++--- tests/test_client.py | 99 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/cbspy/client.py b/src/cbspy/client.py index b6c54fe..dc22867 100644 --- a/src/cbspy/client.py +++ b/src/cbspy/client.py @@ -90,6 +90,7 @@ def get_data( table_id: str, periods: list[str] | None = None, filters: dict[str, list[str]] | str | None = None, + columns: list[str] | None = None, ) -> pl.DataFrame: """Fetch dataset as a Polars DataFrame with human-readable column names. @@ -99,6 +100,9 @@ def get_data( filters: Optional dimension filters. Either a raw OData $filter string or a dict mapping dimension keys to value lists, e.g. ``{"RegioS": ["GM0363", "GM0599"]}``. + columns: Optional list of columns to retrieve. Accepts human-readable + names (e.g. "Total population") or CBS keys (e.g. "TotalPopulation_1"). + Fetches all columns if None. Returns: Polars DataFrame with resolved column names and decoded periods. @@ -108,15 +112,20 @@ def get_data( period_keys = {p["Key"] for p in prop_rows if p.get("Type") == "TimeDimension"} filter_str = self._build_filter(periods, filters, period_keys) - params: dict[str, str] | None = None + params: dict[str, str] = {} if filter_str: - params = {"$filter": filter_str} + params["$filter"] = filter_str - data_rows = self._odata.get_json(table_id, "TypedDataSet", params=params) + if columns is not None: + title_to_key = {v: k for k, v in column_map.items()} + select_keys = [title_to_key.get(c, c) for c in columns] + params["$select"] = ",".join(select_keys) + + data_rows = self._odata.get_json(table_id, "TypedDataSet", params=params or None) if not data_rows: - columns = {column_map.get(k, k): [] for k in column_map} - return pl.DataFrame(columns) + empty_cols = {column_map.get(k, k): [] for k in column_map} + return pl.DataFrame(empty_cols) renamed_rows = [] for row in data_rows: diff --git a/tests/test_client.py b/tests/test_client.py index 6c817bd..ae5d487 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -380,6 +380,105 @@ def test_no_filters(self): assert "$filter" not in sent_params +class TestGetDataColumns: + """Tests for the columns ($select) parameter on get_data().""" + + @respx.mock + def test_columns_sends_select_param(self): + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Periods", "Title": "Periods", "Type": "TimeDimension"}, + { + "Key": "TotalPopulation_1", + "Title": "Total population", + "Type": "Topic", + "Datatype": "Double", + "Unit": "number", + }, + {"Key": "Males_2", "Title": "Males", "Type": "Topic", "Datatype": "Long", "Unit": "number"}, + ] + }, + ) + ) + route = respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( + return_value=httpx.Response( + 200, + json={"value": [{"ID": 0, "Periods": "2023JJ00", "TotalPopulation_1": 17811291}]}, + ) + ) + with Client() as client: + client.get_data("37296eng", columns=["Periods", "Total population"]) + sent_params = dict(route.calls[0].request.url.params) + assert "$select" in sent_params + select_val = sent_params["$select"] + assert "Periods" in select_val + assert "TotalPopulation_1" in select_val + assert "Males_2" not in select_val + + @respx.mock + def test_columns_with_cbs_keys(self): + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Periods", "Title": "Periods", "Type": "TimeDimension"}, + { + "Key": "TotalPopulation_1", + "Title": "Total population", + "Type": "Topic", + "Datatype": "Double", + "Unit": "number", + }, + ] + }, + ) + ) + route = respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( + return_value=httpx.Response( + 200, + json={"value": [{"ID": 0, "Periods": "2023JJ00", "TotalPopulation_1": 17811291}]}, + ) + ) + with Client() as client: + client.get_data("37296eng", columns=["Periods", "TotalPopulation_1"]) + sent_params = dict(route.calls[0].request.url.params) + assert "TotalPopulation_1" in sent_params["$select"] + + @respx.mock + def test_columns_none_sends_no_select(self): + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Periods", "Title": "Periods", "Type": "TimeDimension"}, + { + "Key": "TotalPopulation_1", + "Title": "Total population", + "Type": "Topic", + "Datatype": "Double", + "Unit": "number", + }, + ] + }, + ) + ) + route = respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( + return_value=httpx.Response( + 200, + json={"value": [{"ID": 0, "Periods": "2023JJ00", "TotalPopulation_1": 17811291}]}, + ) + ) + with Client() as client: + client.get_data("37296eng") + sent_params = dict(route.calls[0].request.url.params) + assert "$select" not in sent_params + + class TestClientLifecycle: def test_close_closes_owned_http_client(self): client = Client() From 9afd6dacfb8e0a716b5130f5554dadb54a5dfdc1 Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 17:03:04 +0100 Subject: [PATCH 5/7] feat: add UntypedDataSet support via typed=False When typed=False, fetch UntypedDataSet to preserve CBS statistical symbols (. = not applicable, x = suppressed, - = nil) instead of replacing them with null. Co-Authored-By: Claude Opus 4.6 --- src/cbspy/client.py | 7 +++- tests/test_client.py | 92 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/src/cbspy/client.py b/src/cbspy/client.py index dc22867..1ddca28 100644 --- a/src/cbspy/client.py +++ b/src/cbspy/client.py @@ -91,6 +91,7 @@ def get_data( periods: list[str] | None = None, filters: dict[str, list[str]] | str | None = None, columns: list[str] | None = None, + typed: bool = True, ) -> pl.DataFrame: """Fetch dataset as a Polars DataFrame with human-readable column names. @@ -103,6 +104,9 @@ def get_data( columns: Optional list of columns to retrieve. Accepts human-readable names (e.g. "Total population") or CBS keys (e.g. "TotalPopulation_1"). Fetches all columns if None. + typed: If True (default), fetch TypedDataSet where CBS replaces statistical + symbols with null. If False, fetch UntypedDataSet preserving symbols + like "." (not applicable), "x" (suppressed), "-" (nil). Returns: Polars DataFrame with resolved column names and decoded periods. @@ -121,7 +125,8 @@ def get_data( select_keys = [title_to_key.get(c, c) for c in columns] params["$select"] = ",".join(select_keys) - data_rows = self._odata.get_json(table_id, "TypedDataSet", params=params or None) + resource = "TypedDataSet" if typed else "UntypedDataSet" + data_rows = self._odata.get_json(table_id, resource, params=params or None) if not data_rows: empty_cols = {column_map.get(k, k): [] for k in column_map} diff --git a/tests/test_client.py b/tests/test_client.py index ae5d487..b4ca533 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -479,6 +479,98 @@ def test_columns_none_sends_no_select(self): assert "$select" not in sent_params +class TestGetDataUntyped: + """Tests for typed=False (UntypedDataSet) support.""" + + @respx.mock + def test_typed_false_fetches_untyped_dataset(self): + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Periods", "Title": "Periods", "Type": "TimeDimension"}, + { + "Key": "TotalPopulation_1", + "Title": "Total population", + "Type": "Topic", + "Datatype": "Double", + "Unit": "number", + }, + ] + }, + ) + ) + route = respx.get(f"{BASE}/ODataFeed/OData/37296eng/UntypedDataSet").mock( + return_value=httpx.Response( + 200, + json={"value": [{"ID": 0, "Periods": "2023JJ00", "TotalPopulation_1": "."}]}, + ) + ) + with Client() as client: + df = client.get_data("37296eng", typed=False) + assert df["Total population"][0] == "." + assert route.called + + @respx.mock + def test_typed_true_fetches_typed_dataset(self): + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Periods", "Title": "Periods", "Type": "TimeDimension"}, + { + "Key": "TotalPopulation_1", + "Title": "Total population", + "Type": "Topic", + "Datatype": "Double", + "Unit": "number", + }, + ] + }, + ) + ) + route = respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( + return_value=httpx.Response( + 200, + json={"value": [{"ID": 0, "Periods": "2023JJ00", "TotalPopulation_1": 17811291}]}, + ) + ) + with Client() as client: + client.get_data("37296eng", typed=True) + assert route.called + + @respx.mock + def test_typed_default_is_true(self): + respx.get(f"{BASE}/ODataFeed/OData/37296eng/DataProperties").mock( + return_value=httpx.Response( + 200, + json={ + "value": [ + {"Key": "Periods", "Title": "Periods", "Type": "TimeDimension"}, + { + "Key": "TotalPopulation_1", + "Title": "Total population", + "Type": "Topic", + "Datatype": "Double", + "Unit": "number", + }, + ] + }, + ) + ) + route = respx.get(f"{BASE}/ODataFeed/OData/37296eng/TypedDataSet").mock( + return_value=httpx.Response( + 200, + json={"value": [{"ID": 0, "Periods": "2023JJ00", "TotalPopulation_1": 17811291}]}, + ) + ) + with Client() as client: + client.get_data("37296eng") + assert route.called + + class TestClientLifecycle: def test_close_closes_owned_http_client(self): client = Client() From 689cd968d6842e7288f29dacbabafee7f58f55b8 Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 17:04:57 +0100 Subject: [PATCH 6/7] docs: update for P2 features and fix stale P1 references Add documentation for filters, columns, typed parameters. Add half-yearly period format. Remove references to deleted dutch_name and display_name fields. Co-Authored-By: Claude Opus 4.6 --- docs/examples/exploring-cbs-data.md | 2 +- docs/examples/period-codes.md | 11 ++++++-- docs/examples/population-data.md | 1 + docs/getting-started.md | 44 +++++++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/docs/examples/exploring-cbs-data.md b/docs/examples/exploring-cbs-data.md index 7b958be..3340f62 100644 --- a/docs/examples/exploring-cbs-data.md +++ b/docs/examples/exploring-cbs-data.md @@ -85,7 +85,7 @@ List the available columns: ```python for col in meta.properties[:10]: unit = f" ({col.unit})" if col.unit else "" - print(f" {col.display_name}{unit}") + print(f" {col.name}{unit}") ``` ``` diff --git a/docs/examples/period-codes.md b/docs/examples/period-codes.md index c6f2095..3e64f39 100644 --- a/docs/examples/period-codes.md +++ b/docs/examples/period-codes.md @@ -9,6 +9,7 @@ CBS encodes time periods as strings with a specific format: | Pattern | Example | Decoded | |---------|---------|---------| | `YYYYJJnn` | `2023JJ00` | `2023` (yearly) | +| `YYYYHJnn` | `2023HJ01` | `2023 H1` (half-yearly) | | `YYYYKWnn` | `2023KW01` | `2023 Q1` (quarterly) | | `YYYYMMnn` | `2023MM03` | `2023 March` (monthly) | @@ -25,6 +26,7 @@ _MONTHS = [ "July", "August", "September", "October", "November", "December", ] _YEARLY = re.compile(r"^(\d{4})JJ00$") +_HALFYEARLY = re.compile(r"^(\d{4})HJ0([12])$") _QUARTERLY = re.compile(r"^(\d{4})KW0([1-4])$") _MONTHLY = re.compile(r"^(\d{4})MM(\d{2})$") @@ -33,6 +35,8 @@ def decode_period(raw: str) -> str: s = raw.strip() if m := _YEARLY.match(s): return m.group(1) + if m := _HALFYEARLY.match(s): + return f"{m.group(1)} H{m.group(2)}" if m := _QUARTERLY.match(s): return f"{m.group(1)} Q{m.group(2)}" if m := _MONTHLY.match(s): @@ -54,6 +58,7 @@ mo.md(f"**Input:** `{period_input.value}` **→** **Output:** `{decoded}`") Try these examples: - `2023JJ00` (yearly) +- `2023HJ01` (half-yearly -- H1) - `2023KW03` (quarterly -- Q3) - `2023MM12` (monthly -- December) - `1990JJ00` (older data) @@ -65,6 +70,7 @@ Here is a batch of period codes and their decoded values: ```python {marimo} examples = [ "2023JJ00", "2022JJ00", "2021JJ00", + "2023HJ01", "2023HJ02", "2023KW01", "2023KW02", "2023KW03", "2023KW04", "2023MM01", "2023MM06", "2023MM12", "unknown_format", @@ -78,8 +84,9 @@ mo.ui.table(rows, selection=None) cbspy uses three regular expressions to match period codes: 1. **Yearly:** `^\d{4}JJ00$` -- matches codes like `2023JJ00`. Extracts the four-digit year. -2. **Quarterly:** `^\d{4}KW0[1-4]$` -- matches codes like `2023KW01`. Extracts year and quarter number. -3. **Monthly:** `^\d{4}MM\d{2}$` -- matches codes like `2023MM03`. Extracts year and month number, then maps to the month name. +2. **Half-yearly:** `^\d{4}HJ0[12]$` -- matches codes like `2023HJ01`. Extracts year and half number. +3. **Quarterly:** `^\d{4}KW0[1-4]$` -- matches codes like `2023KW01`. Extracts year and quarter number. +4. **Monthly:** `^\d{4}MM\d{2}$` -- matches codes like `2023MM03`. Extracts year and month number, then maps to the month name. If none of the patterns match, the original string is returned unchanged. This ensures that unexpected formats pass through without raising errors. diff --git a/docs/examples/population-data.md b/docs/examples/population-data.md index 0187da7..50baa4b 100644 --- a/docs/examples/population-data.md +++ b/docs/examples/population-data.md @@ -104,6 +104,7 @@ CBS uses specific period code formats. Pass them directly to `periods`: | Format | Example | Meaning | |--------|---------|---------| | Yearly | `2023JJ00` | Year 2023 | +| Half-yearly | `2023HJ01` | 2023 H1 | | Quarterly | `2023KW01` | 2023 Q1 | | Monthly | `2023MM03` | 2023 March | diff --git a/docs/getting-started.md b/docs/getting-started.md index d4059a3..507239a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -56,7 +56,7 @@ print(f"Period: {meta.period}") print() for col in meta.properties: - print(f" {col.display_name} ({col.unit})") + print(f" {col.name} ({col.unit})") ``` ``` @@ -70,7 +70,7 @@ Period: 1950 - 2023 ... ``` -Each `Column` object has `id`, `name`, `dutch_name`, `unit`, `datatype`, `description`, and a `display_name` computed property that prefers the English name. +Each `Column` object has `id`, `name`, `unit`, `datatype`, and `description`. ## Fetching data @@ -95,6 +95,46 @@ shape: (2, 28) Period codes like `2022JJ00` are automatically decoded to `2022`. Quarterly periods (`2023KW01`) become `2023 Q1`, and monthly periods (`2023MM03`) become `2023 March`. +## Filtering by dimension + +By default, `get_data()` downloads the entire table. Use `periods` to filter by time, or `filters` to filter on any dimension: + +```python +# Filter by region and period +df = client.get_data( + "71450ned", + filters={"RegioS": ["GM0363", "GM0599"]}, + periods=["2023JJ00"], +) +``` + +You can also pass a raw OData filter string: + +```python +df = client.get_data("71450ned", filters="RegioS eq 'GM0363'") +``` + +## Selecting columns + +Many CBS tables have dozens of columns. Use `columns` to fetch only what you need: + +```python +df = client.get_data( + "37296eng", + columns=["Periods", "Total population", "Males"], +) +``` + +You can use either human-readable column names or CBS internal keys. + +## Preserving statistical symbols + +By default, CBS replaces missing-data symbols with `null`. To preserve the original symbols (`.` = not applicable, `x` = suppressed, `-` = nil), pass `typed=False`: + +```python +df = client.get_data("37296eng", typed=False) +``` + ## Error handling cbspy raises specific exceptions for common problems: From 5674a38a442751516c1dd51bea83f13cf7cbe34d Mon Sep 17 00:00:00 2001 From: Thomas Pinder Date: Sat, 21 Feb 2026 17:06:18 +0100 Subject: [PATCH 7/7] Add updated report --- reports/audit-2026-02-21.md | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/reports/audit-2026-02-21.md b/reports/audit-2026-02-21.md index c2b5f5b..30f4823 100644 --- a/reports/audit-2026-02-21.md +++ b/reports/audit-2026-02-21.md @@ -51,14 +51,7 @@ The largest opportunity: cbspy currently exposes only raw dimension codes (e.g. ## Priority 2 — High-Value Data Science Enhancements -### 2.1 Dimension code resolution (labels for region, gender, age, etc.) - -| | | -|---|---| -| **Why it matters** | This is the single biggest usability gap. When a user calls `get_data("71450ned")`, dimension columns contain opaque codes like `"T001038"` (gender) or `"GM0363 "` (municipality). To get "Amsterdam" or "Male", users must manually discover dimension endpoints, fetch them, strip trailing spaces from keys, and join. The existing `cbsodata` library does this automatically; cbspy does not. Every data scientist hitting CBS data hits this wall. | -| **Action** | (a) Add `Client.get_dimension(table_id, dimension_name) -> pl.DataFrame` to fetch and expose dimension code→label mappings. (b) Add a `resolve_labels=True` parameter to `get_data()` that auto-fetches all dimension endpoints and replaces codes with titles. (c) Strip trailing whitespace from dimension keys (CBS pads them to fixed width). | - -### 2.2 Filtering on any dimension, not just periods +### 2.1 Filtering on any dimension, not just periods | | | |---|---| @@ -66,14 +59,14 @@ The largest opportunity: cbspy currently exposes only raw dimension codes (e.g. | **Why it matters** | `get_data()` only accepts a `periods` filter. Data scientists routinely need to filter by region (`RegioS eq 'GM0363'`), gender, age group, etc. Without server-side filtering, users must download entire tables (potentially millions of rows) and filter client-side. | | **Action** | Add a `filters: dict[str, list[str]] | str | None` parameter. Accept either a raw OData `$filter` string or a dict like `{"RegioS": ["GM0363", "GM0599"], "Perioden": ["2023JJ00"]}` that gets compiled to OData syntax. | -### 2.3 Column selection (`$select`) +### 2.2 Column selection (`$select`) | | | |---|---| | **Why it matters** | Many CBS tables have 50+ topic columns. A population researcher may only need 3. Without `$select`, the full payload is transferred and parsed. For large tables this wastes bandwidth and memory. | | **Action** | Add a `columns: list[str] | None` parameter to `get_data()`. Map human-readable names back to CBS keys, then pass as `$select`. | -### 2.4 Switch from ODataApi to ODataFeed +### 2.3 Switch from ODataApi to ODataFeed | | | |---|---| @@ -81,14 +74,14 @@ The largest opportunity: cbspy currently exposes only raw dimension codes (e.g. | **Why it matters** | cbspy uses `/ODataApi/odata/` which returns max 10,000 rows per page. CBS also provides `/ODataFeed/OData/` with no page-size limit — same endpoints, same params, fewer round-trips. For a 500K-row table, ODataApi requires 50+ HTTP requests; ODataFeed may need 1-5. `cbsodata` uses ODataFeed by default. | | **Action** | Change `_ODATA_API = "/ODataFeed/OData"`. This is a one-line change with large performance impact. Add an option to fall back to ODataApi if needed. | -### 2.5 `UntypedDataSet` support +### 2.4 `UntypedDataSet` support | | | |---|---| | **Why it matters** | `TypedDataSet` replaces CBS statistical symbols with `null`, losing information about *why* data is missing (`.` = not applicable, `x` = suppressed, `-` = nil). Economists doing data quality analysis or imputation need these symbols. | | **Action** | Add `typed: bool = True` parameter to `get_data()`. When `False`, fetch `UntypedDataSet` and return string columns preserving symbols. | -### 2.6 Period decoder missing half-yearly format +### 2.5 Period decoder missing half-yearly format | | | |---|---| @@ -239,12 +232,11 @@ The largest opportunity: cbspy currently exposes only raw dimension codes (e.g. | 1.2 | Add context manager to `Client` | P1 | Bug | Small | | 1.3 | Fix or remove broken Dockerfile | P1 | Bug | Trivial | | 1.4 | Implement retry logic for 5xx/network errors | P1 | Bug | Medium | -| 2.1 | Dimension code→label resolution | P2 | Enhancement | Large | -| 2.2 | Filter on any dimension | P2 | Enhancement | Medium | -| 2.3 | `$select` column subsetting | P2 | Enhancement | Small | -| 2.4 | Switch to ODataFeed for larger page size | P2 | Enhancement | Trivial | -| 2.5 | UntypedDataSet support | P2 | Enhancement | Medium | -| 2.6 | Half-yearly period format | P2 | Bug | Trivial | +| 2.1 | Filter on any dimension | P2 | Enhancement | Medium | +| 2.2 | `$select` column subsetting | P2 | Enhancement | Small | +| 2.3 | Switch to ODataFeed for larger page size | P2 | Enhancement | Trivial | +| 2.4 | UntypedDataSet support | P2 | Enhancement | Medium | +| 2.5 | Half-yearly period format | P2 | Bug | Trivial | | 3.1 | PyPI trusted publishing (OIDC) | P3 | CI/CD | Small | | 3.2 | Dynamic versioning with hatch-vcs | P3 | CI/CD | Medium | | 3.3 | Add Dependabot config | P3 | CI/CD | Trivial |