feat(offline): pack first trains with fractional clocks

src/lib/offline/audit_journeys.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Report holes in packed journeys and the raw downloads they were built from.
+
+Read-only. Does not invent fares, platforms, or clocks.
+
+    python3 src/lib/offline/audit_journeys.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+DATA = ROOT / "data" / "en"
+MODES = ("least-distance", "minimum-interchange")
+
+
+@dataclass
+class Audit:
+    origins: int = 0
+    destinations: int = 0
+    with_changes: int = 0
+    distance_only: int = 0
+    missing_first_distance: list[str] = field(default_factory=list)
+    missing_first_changes: list[str] = field(default_factory=list)
+    untyped_fare_distance: list[str] = field(default_factory=list)
+    untyped_fare_changes: list[str] = field(default_factory=list)
+    missing_fare_files: list[str] = field(default_factory=list)
+    platform_gap_distance: int = 0
+    platform_gap_changes: int = 0
+
+    def as_dict(self) -> dict:
+        return {
+            "origins": self.origins,
+            "destinationRows": self.destinations,
+            "withDistinctChanges": self.with_changes,
+            "distanceOnly": self.distance_only,
+            "missingFirstTrain": {
+                "leastDistance": len(self.missing_first_distance),
+                "fewestChanges": len(self.missing_first_changes),
+                "samples": self.missing_first_distance[:10],
+            },
+            "untypedFare": {
+                "leastDistance": len(self.untyped_fare_distance),
+                "fewestChanges": len(self.untyped_fare_changes),
+                "pairs": self.untyped_fare_distance + self.untyped_fare_changes,
+            },
+            "missingFareFiles": self.missing_fare_files,
+            "ridesMissingSomePlatform": {
+                "leastDistance": self.platform_gap_distance,
+                "fewestChanges": self.platform_gap_changes,
+            },
+        }
+
+
+def main() -> int:
+    audit = run_audit()
+    json.dump(audit.as_dict(), sys.stdout, indent=2, ensure_ascii=False)
+    sys.stdout.write("\n")
+    # Fail only on gaps the packer can close. Untyped fares and empty platforms
+    # stay as upstream reports when DMRC returns HTTP 500 or omits platform_name.
+    fixable = len(audit.missing_first_distance) + len(audit.missing_first_changes)
+    if audit.origins == 0 or audit.destinations == 0:
+        return 1
+    return 1 if fixable else 0
+
+
+def run_audit() -> Audit:
+    audit = Audit()
+    journeys = DATA / "journeys"
+    packs = sorted(journeys.glob("*.json"))
+    audit.origins = len(packs)
+    for pack_path in packs:
+        origin = pack_path.stem
+        pack = read_json(pack_path)
+        if not isinstance(pack, dict):
+            continue
+        for destination, row in pack.items():
+            if not isinstance(row, dict):
+                continue
+            audit.destinations += 1
+            distance = row.get("d")
+            changes = row.get("c")
+            if isinstance(distance, dict) and isinstance(changes, dict):
+                audit.with_changes += 1
+            elif isinstance(distance, dict):
+                audit.distance_only += 1
+            pair = f"{origin}->{destination}"
+            inspect_ride(audit, pair, distance, "distance")
+            inspect_ride(audit, pair, changes, "changes")
+    audit.missing_fare_files = missing_fare_files()
+    return audit
+
+
+def inspect_ride(audit: Audit, pair: str, ride: object, kind: str) -> None:
+    if not isinstance(ride, dict):
+        return
+    if "f" not in ride:
+        if kind == "distance":
+            audit.missing_first_distance.append(pair)
+        else:
+            audit.missing_first_changes.append(pair)
+    if "a" in ride and "w" not in ride:
+        if kind == "distance":
+            audit.untyped_fare_distance.append(pair)
+        else:
+            audit.untyped_fare_changes.append(pair)
+    legs = ride.get("g")
+    if isinstance(legs, list) and any(isinstance(leg, dict) and not leg.get("p") for leg in legs):
+        if kind == "distance":
+            audit.platform_gap_distance += 1
+        else:
+            audit.platform_gap_changes += 1
+
+
+def missing_fare_files() -> list[str]:
+    missing: list[str] = []
+    routes_root = DATA / "routes"
+    fares_root = DATA / "fares"
+    if not routes_root.is_dir():
+        return missing
+    for mode in MODES:
+        mode_routes = routes_root / mode
+        if not mode_routes.is_dir():
+            continue
+        for origin_dir in sorted(mode_routes.iterdir()):
+            if not origin_dir.is_dir():
+                continue
+            for route_file in sorted(origin_dir.glob("*.json")):
+                fare = fares_root / mode / origin_dir.name / route_file.name
+                if not fare.is_file():
+                    missing.append(f"{mode}/{origin_dir.name}/{route_file.stem}")
+    return missing
+
+
+def read_json(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/lib/offline/build_journeys.py

@@ -8,7 +8,8 @@
 DATA = ROOT / "data" / "en"
 OUT = DATA / "journeys"
 MODES = ("least-distance", "minimum-interchange")
-CLOCK = __import__("re").compile(r"^\d{2}:[0-5]\d:[0-5]\d$")
+# DMRC sometimes sends fractional seconds (e.g. 06:22:07.500000). Keep wall HH:MM:SS.
+CLOCK = __import__("re").compile(r"^\d{2}:[0-5]\d:[0-5]\d(?:\.\d+)?$")
 
 
 def main() -> None:
@@ -145,7 +146,7 @@ def duration_minutes(value):
 def clock(value):
     if not isinstance(value, str) or CLOCK.fullmatch(value) is None:
         return None
-    return value
+    return value.split(".", 1)[0]
 
 
 def text(value):

src/lib/offline/retry_missing_fares.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Retry only missing new_fare_with_route files (local collector).
+
+Uses the same Origin/Referer as the route download script. The app never calls this.
+
+    python3 src/lib/offline/retry_missing_fares.py
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+DATA = ROOT / "data" / "en"
+BASE = "https://backend.delhimetrorail.com/api/v2"
+MODES = ("least-distance", "minimum-interchange")
+
+
+def main() -> int:
+    from audit_journeys import missing_fare_files
+
+    missing = missing_fare_files()
+    if not missing:
+        print("no missing fare files", flush=True)
+        return 0
+    print(f"retrying {len(missing)} fare files", flush=True)
+    ok = skip = fail = 0
+    for entry in missing:
+        mode, origin, destination = entry.split("/", 2)
+        rel = DATA / "fares" / mode / origin / f"{destination}.json"
+        if rel.is_file() and rel.stat().st_size > 2:
+            skip += 1
+            continue
+        api = f"en/new_fare_with_route/{origin}/{destination}/{mode}/"
+        status, payload = fetch_json(api)
+        if status == "200" and isinstance(payload, dict):
+            rel.parent.mkdir(parents=True, exist_ok=True)
+            temporary = rel.with_suffix(".json.partial")
+            temporary.write_text(json.dumps(payload, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+            temporary.replace(rel)
+            ok += 1
+            print(f"  ok {entry}", flush=True)
+        else:
+            fail += 1
+            print(f"  {status} {entry}", flush=True)
+            time.sleep(0.5)
+    print(f"finished ok={ok} skip={skip} fail={fail}", flush=True)
+    return 0 if fail == 0 else 1
+
+
+def fetch_json(path: str) -> tuple[str, object | None]:
+    url = f"{BASE}/{path}"
+    proc = subprocess.run(
+        [
+            "curl",
+            "-sS",
+            "--max-time",
+            "30",
+            "-w",
+            "\n%{http_code}",
+            "-A",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36",
+            "-H",
+            "Accept: application/json",
+            "-H",
+            "Content-Type: application/json",
+            "-H",
+            "Origin: https://delhimetrorail.com",
+            "-H",
+            "Referer: https://delhimetrorail.com/",
+            url,
+        ],
+        capture_output=True,
+    )
+    if proc.returncode != 0:
+        return "error", None
+    text = proc.stdout.decode("utf-8", errors="replace")
+    if "\n" not in text:
+        return "error", None
+    body, status = text.rsplit("\n", 1)
+    if status != "200":
+        return status, None
+    stripped = body.lstrip()
+    if not stripped.startswith("{") and not stripped.startswith("["):
+        return "html", None
+    try:
+        return "200", json.loads(body)
+    except json.JSONDecodeError:
+        return "json", None
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/lib/offline/test_audit_journeys.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Unit checks for journey pack audit helpers (no network)."""
+
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import audit_journeys
+
+
+class AuditHelpersTest(unittest.TestCase):
+    def test_inspect_ride_counts_untyped_fare_and_missing_first(self) -> None:
+        audit = audit_journeys.Audit()
+        audit_journeys.inspect_ride(
+            audit,
+            "A->B",
+            {"g": [{"n": "Yellow", "c": ["A", "B"]}], "a": 20, "m": 9},
+            "distance",
+        )
+        self.assertEqual(audit.untyped_fare_distance, ["A->B"])
+        self.assertEqual(audit.missing_first_distance, ["A->B"])
+        self.assertEqual(audit.platform_gap_distance, 1)
+
+    def test_missing_fare_files_lists_route_without_fare(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            routes = root / "routes" / "least-distance" / "RI"
+            fares = root / "fares" / "least-distance" / "RI"
+            routes.mkdir(parents=True)
+            fares.mkdir(parents=True)
+            (routes / "KG.json").write_text("{}", encoding="utf-8")
+            (routes / "RCK.json").write_text("{}", encoding="utf-8")
+            (fares / "KG.json").write_text('{"weekday_fare":10,"weekend_fare":10}', encoding="utf-8")
+            with mock.patch.object(audit_journeys, "DATA", root):
+                missing = audit_journeys.missing_fare_files()
+            self.assertEqual(missing, ["least-distance/RI/RCK"])
+
+
+if __name__ == "__main__":
+    unittest.main()

src/lib/offline/test_build_journeys.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""Unit checks for journey packing helpers (no network)."""
+
+from __future__ import annotations
+
+import unittest
+
+from build_journeys import clock, train_pair
+
+
+class ClockTest(unittest.TestCase):
+    def test_accepts_whole_seconds(self) -> None:
+        self.assertEqual(clock("06:01:49"), "06:01:49")
+        self.assertEqual(clock("23:59:59"), "23:59:59")
+
+    def test_truncates_fractional_seconds(self) -> None:
+        self.assertEqual(clock("06:22:07.500000"), "06:22:07")
+        self.assertEqual(clock("06:30:15.5"), "06:30:15")
+
+    def test_rejects_invalid(self) -> None:
+        self.assertIsNone(clock(None))
+        self.assertIsNone(clock(""))
+        self.assertIsNone(clock("6:01:49"))
+        self.assertIsNone(clock("06:61:00"))
+        self.assertIsNone(clock("not-a-time"))
+
+
+class TrainPairTest(unittest.TestCase):
+    def test_first_train_with_fractional_arrive(self) -> None:
+        payload = {
+            "first_train": {
+                "first_train_route_detail": [{"start_time": "06:01:49", "end_time": "06:22:07.500000"}],
+                "endstation_from_first_train_estimated_time": "06:22:07.500000",
+            }
+        }
+        self.assertEqual(
+            train_pair(
+                payload,
+                "first_train",
+                "first_train_route_detail",
+                "endstation_from_first_train_estimated_time",
+            ),
+            ["06:01:49", "06:22:07"],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()