From 47ad76baeb3d6d6bc33e3b575d7d0dab73d21074 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Tue, 13 Oct 2026 11:25:27 +0900 Subject: [PATCH 1/2] feat(schema): load 0.x vehicle states and traffic light links as 1.0 records --- docs/schema/v1/index.md | 4 +- t4_devkit/schema/v1/convert_v0.py | 173 ++++++++++++++++++++++++++--- tests/schema/v1/test_convert_v0.py | 111 +++++++++++++++++- 3 files changed, 268 insertions(+), 20 deletions(-) diff --git a/docs/schema/v1/index.md b/docs/schema/v1/index.md index dcbfa8ea..f622ba43 100644 --- a/docs/schema/v1/index.md +++ b/docs/schema/v1/index.md @@ -138,10 +138,12 @@ Provenance { Every 0.x frame becomes a `sample`: the files of all channels with the same frame number (e.g. `00042.jpg` and `00042.pcd.bin`, as the 0.x converter names them) form one frame. 0.x samples, which are annotated frames only, keep their tokens; 0.x samples of the same frame (one per camera image in camera-only datasets) are merged into the earliest one. A frame with a 2D label is a key frame. -Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. Map `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`. +Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC). Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`. +`vehicle_state` rows get `source: vehicle_interface` (where the 0.x converter reads them) and shift `FORWARD`/`HIGH` becomes `DRIVE`; `additional_info.speed` is dropped and the velocities are `null`. The 0.x `traffic_light` table (`primitive_id`) and `traffic_light_instance_map` table (`traffic_light_linestring_id`, which wins if both list a lamp) link a lamp instance to its lanelet2 way (`type=traffic_light`) and have no states: the way becomes that instance's `map_element` (`type: way`), and the 1.0 `traffic_light` table stays empty. Source LiDARs of a concatenation have no frames of their own in 0.x: they take the concatenated channel's format and `fields`, and `frame_count` is the number of concatenated frames whose `.info` file lists points from them. + 1.0 requires some values that 0.x never stored. They are filled with these placeholders (`V0Converter.FALLBACKS`), which keep the dataset loadable but say nothing about it. In particular, `vehicle.model == "unknown"` means the vehicle dimensions are placeholders and must not be used, and `"unknown"` map IDs mean the map is not identified: | Table / block | Field | Placeholder | diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py index 436c506b..f6fd2cc4 100644 --- a/t4_devkit/schema/v1/convert_v0.py +++ b/t4_devkit/schema/v1/convert_v0.py @@ -4,7 +4,9 @@ import hashlib import os.path as osp import re +import warnings from collections import defaultdict +from datetime import datetime, timezone from glob import glob from typing import TYPE_CHECKING, Any, ClassVar @@ -89,6 +91,8 @@ def __init__(self, data_root: PathLike) -> None: } # 0.x samples are annotated frames only; 1.0 has one sample per frame. self.samples, self.sample_of_frame, self.sample_remap = self._frame_samples() + self.infos: dict[str, list[Record]] = {} # filled by `_frame_infos` + self.traffic_light_ways = self._traffic_light_ways() def convert(self) -> dict[SchemaName, list[Record]]: """Convert every table. @@ -131,6 +135,9 @@ def convert(self) -> dict[SchemaName, list[Record]]: self.convert_object_ann(record) for record in v0["object_ann"] ] + [self.convert_surface_ann(record) for record in v0["surface_ann"]] tables[SchemaName.KEYPOINT] = [self.convert_keypoint(record) for record in v0["keypoint"]] + tables[SchemaName.VEHICLE_STATE] = [ + self.convert_vehicle_state(record) for record in v0["vehicle_state"] + ] # after the labels: an instance's count and first / last label are over all its labels annotations = self._instance_annotations(tables) tables[SchemaName.INSTANCE] = [ @@ -182,7 +189,7 @@ def convert_log(self, record: Record, log2map: dict[str, str]) -> Record: "token": record["token"], "logfile": self._none_if_empty(record["logfile"]), "vehicle_token": self._vehicle_token(record), - "data_captured": record["data_captured"], + "data_captured": record["data_captured"] or self._capture_time(record["token"]), "location": self._none_if_empty(record["location"]), "map_token": log2map.get(record["token"]), } @@ -259,14 +266,11 @@ def convert_camera(self, calibration: Record) -> Record: def convert_lidar(self, calibration: Record) -> Record: """From the first frame: its format, its `.info` file (if any) and its header.""" frames = self.frames[calibration["token"]] - first = frames[0] if frames else {} - info_filename = first.get("info_filename") - info = ( - load_json(osp.join(self.data_root, info_filename)) - if info_filename and osp.exists(osp.join(self.data_root, info_filename)) - else {} - ) - if first.get("fileformat") == "pcd": + if not frames: + return self._convert_source_lidar(calibration) + first = frames[0] + info = self._frame_infos(calibration["token"])[0] + if first["fileformat"] == "pcd": filepath = osp.join(self.data_root, first["filename"]) data = pypcd4.PointCloud.from_path(filepath).pc_data container_format = "pcd" @@ -350,8 +354,12 @@ def convert_visibilities(self) -> list[Record]: def convert_instance( self, record: Record, annotations: dict[str, tuple[int, str, str]] ) -> Record: - """`nbr_annotations` and the first / last annotation come from `annotations`: 1.0 counts - 3D boxes and 2D labels, and 0.x tools count them differently (some write 0 for 2D).""" + """A lamp listed in 0.x traffic_light points to its lanelet2 way in `map_element`. + + `nbr_annotations` and the first / last annotation come from `annotations`: 1.0 counts + 3D boxes and 2D labels, and 0.x tools count them differently (some write 0 for 2D). + """ + way_id = self.traffic_light_ways.get(record["token"]) count, first, last = annotations.get(record["token"], (0, None, None)) return { "token": record["token"], @@ -361,7 +369,7 @@ def convert_instance( "nbr_annotations": count, "first_annotation_token": first, "last_annotation_token": last, - "map_element": {"id": None, "type": None}, + "map_element": {"id": way_id, "type": "way" if way_id is not None else None}, } def convert_sample_annotation(self, record: Record, known_visibility: set[str]) -> Record: @@ -396,8 +404,9 @@ def convert_object_ann(self, record: Record) -> Record: "instance_token": record["instance_token"], "category_token": record["category_token"], "attribute_tokens": record["attribute_tokens"], - "bbox": record["bbox"], - "mask": record.get("mask"), + # 0.x leaves bbox out for mask-only labels; 1.0 requires it. + "bbox": record.get("bbox") or self._mask_bbox(record["mask"]), + "mask": self._non_empty_mask(record.get("mask")), "orientation": record.get("orientation"), "number": record.get("number"), "annotation_type": annotation_type, @@ -408,8 +417,6 @@ def convert_object_ann(self, record: Record) -> Record: def convert_surface_ann(self, record: Record) -> Record: """A 0.x semantic mask becomes an `object_ann` without instance; its box is the mask's.""" mask = record["mask"] - rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} - x, y, w, h = (float(v) for v in cocomask.toBbox(rle)) annotation_type, provenance, confidence = self._label_origin(record) return { "token": record["token"], @@ -417,7 +424,7 @@ def convert_surface_ann(self, record: Record) -> Record: "instance_token": None, "category_token": record["category_token"], "attribute_tokens": [], - "bbox": [x, y, x + w, y + h], + "bbox": self._mask_bbox(mask), "mask": mask, "orientation": None, "number": None, @@ -440,6 +447,34 @@ def convert_keypoint(self, record: Record) -> Record: "confidence": confidence, } + # --- planning tables --- + + def convert_vehicle_state(self, record: Record) -> Record: + """0.x vehicle states come from the Autoware vehicle interface (`/vehicle/status/*`). + + `additional_info.speed` (norm of the longitudinal and lateral velocity) is dropped: it can't + be split back into the 1.0 velocities. + """ + shift_state = record.get("shift_state") + # The 0.x devkit names forward gears FORWARD or HIGH; the 0.x converter writes DRIVE. + if shift_state in ("FORWARD", "HIGH"): + shift_state = "DRIVE" + return { + "token": record["token"], + "timestamp": record["timestamp"], + "accel_pedal": record.get("accel_pedal"), + "brake_pedal": record.get("brake_pedal"), + "steer_pedal": record.get("steer_pedal"), + "steering_tire_angle": record.get("steering_tire_angle"), + "steering_wheel_angle": record.get("steering_wheel_angle"), + "shift_state": shift_state, + "indicators": record.get("indicators"), + "longitudinal_velocity": None, + "lateral_velocity": None, + "heading_rate": None, + "source": "vehicle_interface", + } + # --- helpers --- def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: @@ -572,6 +607,110 @@ def _read_projector(self) -> Record: }, } + def _convert_source_lidar(self, calibration: Record) -> Record: + """A LiDAR without frames of its own: a source of a concatenated channel. + + Its points are stored in the concatenated files, so it has their format and fields; it + counts a frame wherever the `.info` file of a concatenated frame lists points from it. + """ + sensor_token = calibration["sensor_token"] + fallbacks = self.FALLBACKS["lidar"] + container_format, fields, frame_count = "bin", [], 0 + for concat in self.v0["calibrated_sensor"]: + infos = self._frame_infos(concat["token"]) + lengths = [ + sum( + s["length"] + for s in info.get("sources", []) + if s["sensor_token"] == sensor_token + ) + for info in infos + ] + if any(lengths): + concat_lidar = self.convert_lidar(concat) + container_format = concat_lidar["container"]["format"] + fields = concat_lidar["fields"] + frame_count = sum(length > 0 for length in lengths) + break + return { + "container": { + "format": container_format, + "path": None, + "frame_count": frame_count, + "compression": fallbacks["compression"], + }, + "fields": fields, + "source_channels": [], + "frame_id": fallbacks["frame_id"], + "is_undistorted": fallbacks["is_undistorted"], + } + + def _capture_time(self, log_token: str) -> str: + """The log's first sample time, ISO 8601 UTC, for logs without `data_captured`.""" + scenes = { + record["token"] for record in self.v0["scene"] if record["log_token"] == log_token + } + timestamp = min( + record["timestamp"] for record in self.v0["sample"] if record["scene_token"] in scenes + ) + return datetime.fromtimestamp(timestamp / 1e6, tz=timezone.utc).isoformat() + + def _traffic_light_ways(self) -> dict[str, int]: + """The lanelet2 way (`type=traffic_light`) each lamp instance shows, by instance token. + + Two 0.x tables hold this link: `traffic_light` (`primitive_id`, #288) and + `traffic_light_instance_map` (`traffic_light_linestring_id`, from the unmerged + `feat/traffic-light-instance-map` branch), which wins when both list a lamp. + `traffic_light` rows without `primitive_id` are skipped, as no 0.x devkit reads them. + """ + ways = {} + skipped = 0 + for record in self.v0["traffic_light"]: + if "primitive_id" in record: + ways[record["instance_token"]] = int(record["primitive_id"]) + else: + skipped += 1 + if skipped: + warnings.warn(f"Skipped {skipped} traffic_light rows without primitive_id") + for record in self.v0["traffic_light_instance_map"]: + ways[record["instance_token"]] = int(record["traffic_light_linestring_id"]) + return ways + + def _frame_infos(self, calibration_token: str) -> list[Record]: + """The 0.x `.info` file of each frame of a channel ({} where there is none), read once.""" + if calibration_token not in self.infos: + self.infos[calibration_token] = [ + load_json(osp.join(self.data_root, frame["info_filename"])) + if frame.get("info_filename") + and osp.exists(osp.join(self.data_root, frame["info_filename"])) + else {} + for frame in self.frames[calibration_token] + ] + return self.infos[calibration_token] + + @staticmethod + def _non_empty_mask(mask: Record | None) -> Record | None: + """None for a mask without pixels. + + Some 0.x tools give box-only labels a placeholder: an empty RLE whose `size` is + (width, height) instead of the (height, width) of real masks. + """ + if mask is None or "counts" not in mask: + return None + rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} + return mask if cocomask.area(rle) > 0 else None + + @staticmethod + def _mask_bbox(mask: Record) -> list[float]: + """Box around a 0.x RLE mask, as (xmin, ymin, xmax, ymax) on pixel edges. + + Same convention as 0.x boxes, which the 0.x converter writes as (x, y, x + w, y + h) + clipped to the image size. Mask `size` is (height, width), as in COCO. + """ + rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} + x, y, w, h = (float(v) for v in cocomask.toBbox(rle)) + return [x, y, x + w, y + h] + @staticmethod def _vehicle_token(log: Record) -> str: """Deterministic token: 0.x has no vehicle table.""" diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py index 6c3b82ee..8a222c63 100644 --- a/tests/schema/v1/test_convert_v0.py +++ b/tests/schema/v1/test_convert_v0.py @@ -1,9 +1,15 @@ from __future__ import annotations +import base64 import json import shutil +from datetime import datetime, timezone from pathlib import Path +import numpy as np +import pytest +from pycocotools import mask as cocomask + from t4_devkit.schema.v1 import SchemaName, V0Converter, load_dataset SAMPLE_V0 = Path(__file__).parents[2] / "sample" / "t4dataset" @@ -65,8 +71,9 @@ def test_load_v0_labels() -> None: for record in surfaces: category = dataset.get(SchemaName.CATEGORY, record.category_token) assert not category.is_thing - xmin, ymin, xmax, ymax = record.bbox - assert 0 <= xmin < xmax <= record.mask.size[1] and 0 <= ymin < ymax <= record.mask.size[0] + # box on pixel edges around the decoded mask, which is (height, width) + rows, cols = np.nonzero(record.mask.decode()) + assert tuple(record.bbox) == (cols.min(), rows.min(), cols.max() + 1, rows.max() + 1) def test_load_v0_autolabel(tmp_path: Path) -> None: @@ -195,3 +202,103 @@ def test_load_v0_labeled_frame_is_key(tmp_path: Path) -> None: ) frame = load_dataset(data_root).get(SchemaName.SAMPLE_DATA, labeled["sample_data_token"]) assert frame.is_key_frame + + +def test_load_v0_vehicle_state() -> None: + """0.x vehicle states convert; the 0.x forward gear name becomes DRIVE.""" + states = load_dataset(SAMPLE_V0).tables[SchemaName.VEHICLE_STATE] + + assert len(states) == 3 + assert all(state.shift_state == "DRIVE" for state in states) + assert all(state.source == "vehicle_interface" for state in states) + + +def test_load_v0_capture_time(tmp_path: Path) -> None: + """An empty 0.x `data_captured` becomes the first sample time.""" + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + log_path = data_root / "annotation" / "log.json" + logs = json.loads(log_path.read_text()) + logs[0]["data_captured"] = "" + log_path.write_text(json.dumps(logs)) + + tables = load_dataset(data_root).tables + first = min(sample.timestamp for sample in tables[SchemaName.SAMPLE]) + expected = datetime.fromtimestamp(first / 1e6, tz=timezone.utc).isoformat() + assert tables[SchemaName.LOG][0].data_captured == expected + + +@pytest.mark.parametrize( + ("table", "way_key"), + [ + ("traffic_light", "primitive_id"), + ("traffic_light_instance_map", "traffic_light_linestring_id"), + ], +) +def test_load_v0_traffic_light(tmp_path: Path, table: str, way_key: str) -> None: + """A 0.x lamp-to-way row sets the instance's lanelet2 way.""" + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + instance_token = json.loads((data_root / "annotation" / "instance.json").read_text())[0][ + "token" + ] + rows = [{"token": "tl0", "instance_token": instance_token, way_key: "1234"}] + (data_root / "annotation" / f"{table}.json").write_text(json.dumps(rows)) + + tables = load_dataset(data_root).tables + assert tables[SchemaName.TRAFFIC_LIGHT] == [] + map_element = tables[SchemaName.INSTANCE][0].map_element + assert (map_element.id, map_element.type) == (1234, "way") + + +def test_load_v0_source_lidar(tmp_path: Path) -> None: + """A source LiDAR takes the concatenated channel's fields and counts frames with its points.""" + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + annotation = data_root / "annotation" + + def edit(name: str, change) -> None: + path = annotation / f"{name}.json" + path.write_text(json.dumps(change(json.loads(path.read_text())))) + + source = {"token": "src", "channel": "LIDAR_SOURCE", "modality": "lidar"} + calibration = {"token": "src_calib", "sensor_token": "src"} + calibration |= {"translation": [0.0, 0.0, 0.0], "rotation": [1.0, 0.0, 0.0, 0.0]} + calibration |= {"camera_intrinsic": [], "camera_distortion": []} + edit("sensor", lambda records: [*records, source]) + edit("calibrated_sensor", lambda records: [*records, calibration]) + + def add_info(records: list[dict]) -> list[dict]: + frames = [r for r in records if r["filename"].startswith("data/LIDAR_TOP")] + for i, record in enumerate(frames): + record["info_filename"] = f"data/LIDAR_TOP_INFO/{i}.json" + # points from the source in the first two frames only + sources = [{"sensor_token": "src", "idx_begin": 0, "length": 10 if i < 2 else 0}] + info = {"num_pts_feats": 7, "sources": sources} + (data_root / record["info_filename"]).parent.mkdir(exist_ok=True) + (data_root / record["info_filename"]).write_text(json.dumps(info)) + return records + + edit("sample_data", add_info) + + sensors = {s.channel: s for s in load_dataset(data_root).tables[SchemaName.SENSOR]} + concat, source_lidar = sensors["LIDAR_TOP"].lidar, sensors["LIDAR_SOURCE"].lidar + assert concat.source_channels == ["LIDAR_SOURCE"] + assert source_lidar.fields == concat.fields and len(concat.fields) == 7 + assert source_lidar.container.frame_count == 2 + + +def test_load_v0_empty_mask(tmp_path: Path) -> None: + """A box-only 0.x label with an empty placeholder mask has no mask in 1.0.""" + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + path = data_root / "annotation" / "object_ann.json" + records = json.loads(path.read_text()) + empty = cocomask.encode(np.zeros((600, 800), dtype=np.uint8, order="F")) + # as written by some 0.x tools: (width, height) + records[0]["mask"] = {"size": [800, 600], "counts": base64.b64encode(empty["counts"]).decode()} + path.write_text(json.dumps(records)) + + annotation = load_dataset(data_root).tables[SchemaName.OBJECT_ANN][0] + assert annotation.mask is None + assert list(annotation.bbox) == records[0]["bbox"] From bd6c15aeaa74629bc50257412fb4e72ac3670484 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Tue, 13 Oct 2026 14:05:11 +0900 Subject: [PATCH 2/2] feat(schema): fill 0.x category index by list position --- docs/schema/v1/index.md | 2 +- t4_devkit/schema/v1/convert_v0.py | 20 +++++++++++++++++--- tests/schema/v1/test_convert_v0.py | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/schema/v1/index.md b/docs/schema/v1/index.md index f622ba43..781ca93b 100644 --- a/docs/schema/v1/index.md +++ b/docs/schema/v1/index.md @@ -140,7 +140,7 @@ Every 0.x frame becomes a `sample`: the files of all channels with the same fram Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC). -Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`. +Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). Categories without `index` get their position in `category.json`, as the 0.x devkit reads them (all set or none). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`. `vehicle_state` rows get `source: vehicle_interface` (where the 0.x converter reads them) and shift `FORWARD`/`HIGH` becomes `DRIVE`; `additional_info.speed` is dropped and the velocities are `null`. The 0.x `traffic_light` table (`primitive_id`) and `traffic_light_instance_map` table (`traffic_light_linestring_id`, which wins if both list a lamp) link a lamp instance to its lanelet2 way (`type=traffic_light`) and have no states: the way becomes that instance's `map_element` (`type: way`), and the 1.0 `traffic_light` table stays empty. Source LiDARs of a concatenation have no frames of their own in 0.x: they take the concatenated channel's format and `fields`, and `frame_count` is the number of concatenated frames whose `.info` file lists points from them. diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py index f6fd2cc4..a0937bcb 100644 --- a/t4_devkit/schema/v1/convert_v0.py +++ b/t4_devkit/schema/v1/convert_v0.py @@ -122,7 +122,7 @@ def convert(self) -> dict[SchemaName, list[Record]]: self.convert_sample_data(record) for record in v0["sample_data"] ] - tables[SchemaName.CATEGORY] = [self.convert_category(record) for record in v0["category"]] + tables[SchemaName.CATEGORY] = self.convert_categories() tables[SchemaName.ATTRIBUTE] = [ self.convert_attribute(record) for record in v0["attribute"] ] @@ -316,13 +316,27 @@ def convert_sample_data(self, record: Record) -> Record: # --- label tables --- - def convert_category(self, record: Record) -> Record: + def convert_categories(self) -> list[Record]: + """Categories without `index` get their list position, as the 0.x devkit reads them. + + Raises: + ValueError: If some categories have an `index` and others do not. + """ + records = self.v0["category"] + indices = [record.get("index") for record in records] + if None in indices: + if any(index is not None for index in indices): + raise ValueError("Category index is not set for some categories.") + indices = list(range(len(records))) + return [self.convert_category(record, index) for record, index in zip(records, indices)] + + def convert_category(self, record: Record, index: int) -> Record: """`is_thing` is false only for categories used by `surface_ann` (road, sky) and nothing else.""" token = record["token"] return { "token": token, "name": record["name"], - "index": record.get("index"), + "index": index, "description": self._none_if_empty(record.get("description")), "is_thing": token in self.thing_categories or token not in self.surface_categories, "has_orientation": record.get("has_orientation", False), diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py index 8a222c63..5f1619fb 100644 --- a/tests/schema/v1/test_convert_v0.py +++ b/tests/schema/v1/test_convert_v0.py @@ -76,6 +76,21 @@ def test_load_v0_labels() -> None: assert tuple(record.bbox) == (cols.min(), rows.min(), cols.max() + 1, rows.max() + 1) +def test_load_v0_category_index(tmp_path: Path) -> None: + """Categories without index get their list position; a partly set index is rejected.""" + categories = load_dataset(SAMPLE_V0).tables[SchemaName.CATEGORY] + assert [category.index for category in categories] == list(range(len(categories))) + + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + category_path = data_root / "annotation" / "category.json" + records = json.loads(category_path.read_text()) + records[0]["index"] = 5 + category_path.write_text(json.dumps(records)) + with pytest.raises(ValueError, match="index"): + load_dataset(data_root) + + def test_load_v0_autolabel(tmp_path: Path) -> None: """A 0.x autolabel becomes annotation_type auto, with the model as provenance.""" data_root = tmp_path / "dataset"