Updates for numpy v2

.abench/check-pr.yaml

@@ -0,0 +1,67 @@
+# Requires abench >= 0.2.0 (data assets, branch/PR sources, and GitHub publishing).
+# Run from the repository root: abench . (choose check-pr.yaml, then answer inputs).
+# Publishes a summary and charts to the selected ActivitySim PR after the suite.
+# Requires gh >= 2.99, gh auth login, and write access to ActivitySim/activitysim.
+# Preview without posting: abench .abench/check-pr.yaml --publish-dry-run
+# Retry posting retained results: abench publish /path/to/suite-output
+schema_version: 1
+inputs:
+  activitysim_pr:
+    type: integer
+    required: true
+    minimum: 1
+    description: ActivitySim PR number to compare against current main
+  households:
+    type: integer
+    default: 500000
+    minimum: 0
+    description: Households to sample; 0 uses the full population
+  processes:
+    type: integer
+    default: 4
+    minimum: 1
+    description: Number of worker processes
+vars:
+  activitysim_repository: ActivitySim/activitysim
+output_root: ../benchmark-runs/mtc-${timestamp}
+publish:
+  github:
+    repository: ${activitysim_repository}
+    pr: ${activitysim_pr}
+    baseline: main
+data_assets:
+  # Match download_external_example(name="prototype_mtc_extended", ...).
+  name: prototype_mtc_extended
+  assets:
+    data_full.tar.zst:
+      url: https://github.com/ActivitySim/activitysim-prototype-mtc/releases/download/v1.3.4/data_full.tar.zst
+      sha256: b402506a61055e2d38621416dd9a5c7e3cf7517c0a9ae5869f6d760c03284ef3
+      # Asset destinations are relative to this YAML and must remain inside .abench.
+      # The archive and extracted contents still reuse ActivitySim's shared cache.
+      unpack: data_full
+defaults:
+  model_dir: ..
+  profile: mtc
+  data_dir: data_full
+  config_overlay: [../configs_explicit_chunk]
+  households: ${households}
+  processes: ${processes}
+  multiprocess: true
+  sharrow: true
+  memory: 32g
+  shm_size: 8g
+  sources:
+    - sharrow=ActivitySim/sharrow@fc175b27d8e0c5d202721c67d96b050e6117b235
+runs:
+  main:
+    label: MTC main — explicit chunking
+    sources:
+      - name: activitysim
+        repository: ${activitysim_repository}
+        branch: main
+  pr:
+    label: MTC PR ${activitysim_pr} — explicit chunking
+    sources:
+      - name: activitysim
+        repository: ${activitysim_repository}
+        pr: ${activitysim_pr}

.github/workflows/ci.yml

@@ -15,12 +15,14 @@ jobs:
   foundation:
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.10"]
+        activitysim: [locked, main]
     defaults:
       run:
         shell: bash -l {0}
-    name: linux-64-py${{ matrix.python-version }}
+    name: linux-64-py${{ matrix.python-version }}-${{ matrix.activitysim }}
     runs-on: ubuntu-latest
     steps:
       # checkout the code in this repository
@@ -36,14 +38,15 @@ jobs:
       - name: "Set up Python"
         uses: actions/setup-python@v5
         with:
-          python-version-file: ".python-version"
+          python-version: ${{ matrix.python-version }}
 
       - name: Install everything
         run: |
           uv sync --locked
 
-      # checkout the main branch of ActivitySim itself
+      # Only the compatibility job replaces the locked release.
       - uses: actions/checkout@v4
+        if: matrix.activitysim == 'main'
         with:
           repository: 'ActivitySim/activitysim'
           ref: main
@@ -52,12 +55,16 @@ jobs:
 
       # install ActivitySim, overriding the one in the workspace,
       # as the pre-installed one is not necessarily the most up-to-date
-      - name: Install ActivitySim
+      - name: Install ActivitySim main
+        if: matrix.activitysim == 'main'
         run: |
           uv pip install --no-deps ./cloned/activitysim
 
       # The next step is to run the tests, but now we need "--no-sync" to avoid
       # automatically reinstalling the overwritten version of ActivitySim
       - name: Test this implementation
+        env:
+          ACTIVITYSIM_TEST_SOURCE: ${{ matrix.activitysim }}
+          UV_NO_SYNC: ${{ matrix.activitysim == 'main' && '1' || '0' }}
         run: |
           uv run --no-sync pytest ./test

.gitignore

@@ -5,3 +5,8 @@
 **/__pycache__
 activitysim.log
 *.tar.zst
+/data_full
+/benchmark-runs/
+
+# Local link to the shared benchmark data cache.
+/.abench/data_full

.python-version

@@ -1 +1 @@
-3.11
+3.10

README.md

@@ -48,6 +48,35 @@ example_dir = download_external_example(
 )
 ```
 
+# Running the Sharrow scripts
+
+From the repository root, use the shared, locked environment:
+
+```sh
+uv run --locked scripts/run-small-sharrow.py
+# Or download the full dataset and run a 500,000-household sample:
+uv run --locked scripts/run-large-sharrow.py
+```
+
+The executable scripts can also be called directly from the repository root.
+They use `pyproject.toml` and `uv.lock`; no separate script environments or
+script lockfiles are needed. Output goes into `scripts/run-small-sharrow-output`
+or `scripts/run-large-sharrow-output`, respectively.
+
+# Testing
+
+Run the regression suite and script smoke tests against the locked release:
+
+```sh
+uv run --locked pytest test
+```
+
+CI runs this suite against both the locked ActivitySim release and current
+ActivitySim `main`. The main job replaces only ActivitySim, keeping the other
+locked dependencies, and uses `--no-sync` to preserve that installation.
+The script smoke tests validate startup, configuration, and filesystem behavior;
+they do not download the full dataset or simulate 500,000 households.
+
 # Benchmarking
 
 The `prototype_mtc` example model is run using the `activitysim` command line tool.

configs_explicit_chunk/README.md

@@ -0,0 +1,41 @@
+# Explicit chunking overlay
+
+Use `-c configs_explicit_chunk -c configs` (or `-c configs_explicit_chunk -c configs_mp -c configs` for multiprocessing).
+For the benchmark use `--config-overlay configs_explicit_chunk`.
+
+This overlay enables explicit chunking and caps supported components at 10,000 chooser rows across the process group (2,500 per worker with four processes). This conservative initial limit leaves room for the shared skims, population tables, and temporary alternatives. It is not a 16 GB hard memory guarantee; row limits do not bound fixed tables, retained allocator pages, or every intermediate in main. [ActivitySim PR #1110](https://github.com/ActivitySim/activitysim/pull/1110) extends several chunk boundaries to cover more of those intermediates. The overlay also runs on ActivitySim 1.6/main without those extensions.
+
+Selection uses the union of components whose whole-container memory exceeded 16,000,000,000 bytes in either previous 500,000-household measured run. Legacy windows are approximate and include overlap with other workers, so these are group peaks, not isolated component allocations. These historical measurements are summarized below; a machine-readable baseline is not included in this repository.
+
+| Component | Highest observed GiB | Overlay / limitation |
+|---|---:|---|
+| school_location | 18.36 | school_location.yaml |
+| workplace_location | 21.90 | workplace_location.yaml |
+| auto_ownership_simulate | 16.95 | No wired explicit-chunk control in these revisions |
+| free_parking | 16.92 | No wired explicit-chunk control in these revisions |
+| cdap_simulate | 16.94 | No wired explicit-chunk control in these revisions |
+| mandatory_tour_frequency | 17.92 | No wired explicit-chunk control in these revisions |
+| mandatory_tour_scheduling | 27.88 | mandatory_tour_scheduling.yaml |
+| joint_tour_frequency | 26.99 | No wired explicit-chunk control in these revisions |
+| joint_tour_participation | 27.02 | No wired explicit-chunk control in these revisions |
+| joint_tour_destination | 27.02 | joint_tour_destination.yaml |
+| joint_tour_scheduling | 22.38 | joint_tour_scheduling.yaml |
+| non_mandatory_tour_frequency | 20.13 | non_mandatory_tour_frequency.yaml |
+| non_mandatory_tour_destination | 21.71 | non_mandatory_tour_destination.yaml |
+| non_mandatory_tour_scheduling | 25.05 | non_mandatory_tour_scheduling.yaml |
+| tour_mode_choice_simulate | 24.02 | tour_mode_choice.yaml |
+| atwork_subtour_frequency | 23.97 | No wired explicit-chunk control in these revisions |
+| atwork_subtour_destination | 25.13 | atwork_subtour_destination.yaml |
+| atwork_subtour_scheduling | 24.69 | tour_scheduling_atwork.yaml |
+| atwork_subtour_mode_choice | 24.54 | tour_mode_choice.yaml |
+| stop_frequency | 24.54 | No wired explicit-chunk control in these revisions |
+| trip_purpose | 21.15 | No wired explicit-chunk control in these revisions |
+| trip_destination | 22.29 | trip_destination.yaml |
+| trip_scheduling | 20.45 | No wired explicit-chunk control in these revisions |
+| trip_mode_choice | 20.39 | trip_mode_choice.yaml |
+| trip_purpose_and_destination | 20.45 | No wired explicit-chunk control in these revisions |
+| write_trip_matrices | 21.10 | No wired explicit-chunk control in these revisions |
+| summarize | 24.95 | No wired explicit-chunk control in these revisions |
+| joint_tour_composition | 19.50 | No wired explicit-chunk control in these revisions |
+
+For unsupported components, adding `explicit_chunk` would either be rejected or silently do nothing. They therefore have no pretend override. In particular, trip scheduling and CDAP use a chunk-by-group helper but do not pass a configurable explicit size; output matrices/summarization also lack this control. No ordinary config or ActivitySim source is modified. At-work mode choice shares `tour_mode_choice.yaml` with tour mode choice and logsum evaluation.

configs_explicit_chunk/atwork_subtour_destination.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/joint_tour_destination.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/joint_tour_scheduling.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/mandatory_tour_scheduling.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/non_mandatory_tour_destination.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/non_mandatory_tour_frequency.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/non_mandatory_tour_scheduling.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/school_location.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/settings.yaml

@@ -0,0 +1,4 @@
+# Enable per-component row limits; leave adaptive memory chunking off.
+inherit_settings: true
+chunk_training_mode: explicit
+chunk_size: 0

configs_explicit_chunk/tour_mode_choice.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/tour_scheduling_atwork.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/trip_destination.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/trip_mode_choice.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

configs_explicit_chunk/workplace_location.yaml

@@ -0,0 +1,4 @@
+# Bound chooser batches for the worker group (2,500 rows each with 4 workers).
+# Remaining model settings and specifications come from configs/.
+inherit_settings: true
+explicit_chunk: 10000

pyproject.toml

@@ -5,13 +5,13 @@ description = "User environment for Prototype MTC Model"
 readme = "README.md"
 requires-python = ">=3.10,<3.12"
 dependencies = [
-    "activitysim==1.4",
+    "activitysim>=1.6",
     "multimethod<2.0",
     "numba>=0.57",
-    "numpy>=1.16.1,<1.26",
+    "numpy>=2,<3",
     "openmatrix>=0.3.5.0",
     "pandas>=2.1.0,<3",
-    "pandera>=0.15,<0.18.1",
+    "pandera>=0.30",
     "psutil>=4.1",
     "pydantic>=2.6",
     "setuptools>=80.9.0,<81",

scripts/run-large-sharrow.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env -S uv run --locked
+
+"""
+Run the MTC example model with sharrow enabled, on a 500,000-household sample of the full dataset.
+
+Run from the repository root with `uv run --locked scripts/run-large-sharrow.py`
+or execute this file directly there. Both use the shared pyproject.toml and uv.lock.
+"""
+
+import os.path
+from pathlib import Path
+
+import platformdirs
+import activitysim.abm  # register components # noqa: F401
+from activitysim.core import workflow
+from activitysim.examples.external import download_asset
+
+import pandas as pd
+
+
+def main():
+    # Opt into pandas' future behavior for object-array fill operations.
+    pd.set_option("future.no_silent_downcasting", True)
+
+    working_dir = Path(__file__).parents[1]
+
+    download_asset(
+        url="https://github.com/ActivitySim/activitysim-prototype-mtc/releases/download/v1.3.4/data_full.tar.zst",
+        target_path=os.path.join(working_dir, "data_full.tar.zst"),
+        sha256="b402506a61055e2d38621416dd9a5c7e3cf7517c0a9ae5869f6d760c03284ef3",
+        link=Path(platformdirs.user_cache_dir(appname="ActivitySim")) / "External-Data",
+        base_path=str(working_dir),
+        unpack="data_full",
+    )
+
+    out_dir = Path(str(__file__).replace(".py", "-output"))
+    out_dir.mkdir(exist_ok=True)
+    out_dir.joinpath(".gitignore").write_text("**\n")
+
+    settings = dict(
+        cleanup_pipeline_after_run=False,
+        treat_warnings_as_errors=False,
+        households_sample_size=500_000,
+        chunk_size=0,
+        use_shadow_pricing=True,
+        sharrow="require",
+        recode_pipeline_columns=True,
+    )
+
+    state = workflow.State.make_default(
+        working_dir=working_dir,
+        configs_dir=("configs",),
+        data_dir="data_full",
+        output_dir=out_dir,
+        settings=settings,
+    )
+    state.filesystem.persist_sharrow_cache()
+    state.logging.config_logger()
+
+    state.run.all(resume_after=None)
+
+
+if __name__ == "__main__":
+    main()