diff --git a/docs/schema/v1/index.md b/docs/schema/v1/index.md index be6e0693..dcbfa8ea 100644 --- a/docs/schema/v1/index.md +++ b/docs/schema/v1/index.md @@ -131,3 +131,27 @@ Provenance { "process_config_version_id": -- Web.Auto processing configuration version. } ``` + +## Loading 0.x Datasets + +`load_dataset()` reads a 0.x dataset (no `manifest.json`) by converting its tables to 1.0 records when loading, on a best-effort basis. Sensor data files stay as they are: one file per frame, and `sample_data.frame_index` is `null`. + +Every 0.x frame becomes a `sample`: the files of all channels with the same frame number (e.g. `00042.jpg` and `00042.pcd.bin`, as the 0.x converter names them) form one frame. 0.x samples, which are annotated frames only, keep their tokens; 0.x samples of the same frame (one per camera image in camera-only datasets) are merged into the earliest one. A frame with a 2D label is a key frame. + +Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. Map `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`. + +Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`. + +1.0 requires some values that 0.x never stored. They are filled with these placeholders (`V0Converter.FALLBACKS`), which keep the dataset loadable but say nothing about it. In particular, `vehicle.model == "unknown"` means the vehicle dimensions are placeholders and must not be used, and `"unknown"` map IDs mean the map is not identified: + +| Table / block | Field | Placeholder | +| --------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `vehicle` | `model` | `"unknown"` | +| `vehicle` | `length`, `width`, `height`, `wheel_base` | `1.0` | +| `vehicle` | `name` | 0.x `log.vehicle`, `null` if empty | +| `map` | `map_id`, `version_id` | `"unknown"` | +| `map` | `source` | `"webauto"` | +| `ego_pose` | `source`, `frame` | `"localization"`, `"map"` | +| `sensor.camera` | `is_rectified` | `true` | +| `sensor.lidar` | `frame_id`, `is_undistorted`, `container.compression` | `"base_link"`, `true`, `"none"` | +| `sensor.lidar` | `fields` (`.bin`) | `x y z intensity channel return_type time_stamp`, cut to `num_pts_feats` (default 5), all `float32` | diff --git a/docs/schema/v1/table.md b/docs/schema/v1/table.md index 379796c7..499a23a5 100644 --- a/docs/schema/v1/table.md +++ b/docs/schema/v1/table.md @@ -316,7 +316,7 @@ attribute { - Filename: `visibility.json` -Unchanged from 0.x. +Unknown visibility is `sample_annotation.visibility_token: null`; the 0.x level `unavailable` is dropped. ```json visibility { @@ -360,7 +360,7 @@ sample_annotation { "sample_token": -- Foreign key to the `Sample` table. "instance_token": -- Foreign key to the `Instance` table. "attribute_tokens": <[str;N]> -- Foreign keys to the `Attribute` table. - "visibility_token": -- Foreign key to the `Visibility` table. + "visibility_token": -- Foreign key to the `Visibility` table. `null` if unknown. "translation": <[float;3]> -- Box center in meters. "size": <[float;3]> -- (width, length, height) in meters. "rotation": <[float;4]> -- Quaternion (w, x, y, z). diff --git a/t4_devkit/common/io.py b/t4_devkit/common/io.py index 4c74c784..9e107d9f 100644 --- a/t4_devkit/common/io.py +++ b/t4_devkit/common/io.py @@ -16,7 +16,11 @@ if TYPE_CHECKING: from t4_devkit.typing import PathLike -__all__ = ("load_json", "save_json", "load_pointcloud", "load_image") +__all__ = ("load_json", "save_json", "load_pointcloud", "load_image", "V0_BIN_FIELDS") + +# 0.x `.bin` layout: this list cut to `num_pts_feats`, all float32 (as written by the converter). +# `.bin` stores no names; these are the 1.0 names (0.x calls `channel` "ring"), all float32. +V0_BIN_FIELDS = ("x", "y", "z", "intensity", "channel", "return_type", "time_stamp") def load_json(filename: PathLike) -> Any: diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index e6434b09..f503382a 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -9,7 +9,7 @@ import numpy as np from attrs import define, field -from t4_devkit.common.io import load_json, load_pointcloud +from t4_devkit.common.io import V0_BIN_FIELDS, load_json, load_pointcloud from t4_devkit.common.timestamp import Stamp if TYPE_CHECKING: @@ -31,9 +31,6 @@ # Point fields of `LidarPointCloud.points`, in row order. XYZI = ("x", "y", "z", "intensity") -# 0.x `.bin` layout: this list cut to `num_pts_feats`, all float32 (as written by the converter). -V0_BIN_FIELDS = ("x", "y", "z", "intensity", "ring", "return_type", "time_stamp") - @define class PointCloudSourceInfo: diff --git a/t4_devkit/schema/v1/__init__.py b/t4_devkit/schema/v1/__init__.py index 92661d70..16b9f4e9 100644 --- a/t4_devkit/schema/v1/__init__.py +++ b/t4_devkit/schema/v1/__init__.py @@ -2,6 +2,7 @@ from .attribute import * # noqa from .caption import * # noqa from .category import * # noqa +from .convert_v0 import * # noqa from .ego_pose import * # noqa from .instance import * # noqa from .keypoint import * # noqa diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py new file mode 100644 index 00000000..436c506b --- /dev/null +++ b/t4_devkit/schema/v1/convert_v0.py @@ -0,0 +1,597 @@ +from __future__ import annotations + +import base64 +import hashlib +import os.path as osp +import re +from collections import defaultdict +from glob import glob +from typing import TYPE_CHECKING, Any, ClassVar + +import pypcd4 +import yaml +from PIL import Image +from pycocotools import mask as cocomask + +from t4_devkit.common.io import V0_BIN_FIELDS, load_json +from t4_devkit.schema.tables.visibility import VisibilityLevel as V0VisibilityLevel + +from .name import SchemaName +from .sensor import DistortionModel + +if TYPE_CHECKING: + from t4_devkit.typing import PathLike + +__all__ = ["V0Converter"] + +Record = dict[str, Any] + + +class V0Converter: + """Read a 0.x dataset as 1.0 table dicts, best effort. + + Only the table files are read (plus one header per LiDAR channel and the map files); sensor + data files are not rewritten, so frames keep one file each and `frame_index` is None. + Values that 0.x never stored come from `FALLBACKS`. + + Examples: + >>> tables = V0Converter(data_root).convert() + """ + + # Values 1.0 requires that 0.x never stored. These placeholders keep a converted dataset + # loadable, they are not facts about the dataset (listed in `docs/schema/v1/index.md`). + # `vehicle.model == "unknown"` marks placeholder vehicle dimensions. + FALLBACKS: ClassVar[dict[str, Any]] = { + "vehicle": { + "model": "unknown", + "length": 1.0, + "width": 1.0, + "height": 1.0, + "wheel_base": 1.0, + }, + "map": {"map_id": "unknown", "version_id": "unknown", "source": "webauto"}, + "map_filename": "map/lanelet2_map.osm", + "map_projector_filename": "map/map_projector_info.yaml", + "ego_pose": {"source": "localization", "frame": "map"}, + "lidar": {"frame_id": "base_link", "is_undistorted": True, "compression": "none"}, + "camera": {"is_rectified": True}, + } + + # The 0.x converter names the files of one frame by its frame number, e.g. `00042.jpg` and + # `00042.pcd.bin`: it groups the channels of a frame (their timestamps differ by up to ~90 ms). + FRAME_NUMBER: ClassVar[re.Pattern] = re.compile(r"^(\d+)\.") + + def __init__(self, data_root: PathLike) -> None: + """ + Args: + data_root (PathLike): Dataset root, i.e. the `/` directory. + """ + self.data_root = data_root + # Every 0.x table file present, by table name; a missing table reads as empty. + self.v0: dict[str, list[Record]] = defaultdict(list) + for filepath in glob(osp.join(data_root, "annotation", "*.json")): + self.v0[osp.splitext(osp.basename(filepath))[0]] = load_json(filepath) + + # Lookups shared by several tables. + self.sensors = {record["token"]: record for record in self.v0["sensor"]} + self.frames: dict[str, list[Record]] = defaultdict(list) + for record in self.v0["sample_data"]: + self.frames[record["calibrated_sensor_token"]].append(record) + self.thing_categories = { + record["category_token"] for record in self.v0["object_ann"] + self.v0["instance"] + } + self.surface_categories = {record["category_token"] for record in self.v0["surface_ann"]} + # A labeled frame is a key frame (0.x has a few labels on non-key frames). + self.labeled_frames = { + record["sample_data_token"] + for table in ("object_ann", "surface_ann", "keypoint") + for record in self.v0[table] + } + # 0.x samples are annotated frames only; 1.0 has one sample per frame. + self.samples, self.sample_of_frame, self.sample_remap = self._frame_samples() + + def convert(self) -> dict[SchemaName, list[Record]]: + """Convert every table. + + Returns: + Raw 1.0 records per table; empty for tables not converted. + """ + v0 = self.v0 + maps = [self.convert_map(record) for record in v0["map"]] + maps = [record for record in maps if record is not None] + log2map = {token: record["token"] for record in maps for token in record["log_tokens"]} + visibilities = self.convert_visibilities() + known_visibility = {record["token"] for record in visibilities} + + tables: dict[SchemaName, list[Record]] = {name: [] for name in SchemaName} + tables[SchemaName.MAP] = maps + tables[SchemaName.VEHICLE] = [self.convert_vehicle(record) for record in v0["log"]] + tables[SchemaName.LOG] = [self.convert_log(record, log2map) for record in v0["log"]] + tables[SchemaName.SCENE] = [self.convert_scene(record) for record in v0["scene"]] + tables[SchemaName.SAMPLE] = self.samples + tables[SchemaName.EGO_POSE] = [self.convert_ego_pose(record) for record in v0["ego_pose"]] + tables[SchemaName.SENSOR] = [ + self.convert_sensor(index, record) + for index, record in enumerate(v0["calibrated_sensor"]) + ] + tables[SchemaName.SAMPLE_DATA] = [ + self.convert_sample_data(record) for record in v0["sample_data"] + ] + + tables[SchemaName.CATEGORY] = [self.convert_category(record) for record in v0["category"]] + tables[SchemaName.ATTRIBUTE] = [ + self.convert_attribute(record) for record in v0["attribute"] + ] + tables[SchemaName.VISIBILITY] = visibilities + tables[SchemaName.SAMPLE_ANNOTATION] = [ + self.convert_sample_annotation(record, known_visibility) + for record in v0["sample_annotation"] + ] + tables[SchemaName.OBJECT_ANN] = [ + self.convert_object_ann(record) for record in v0["object_ann"] + ] + [self.convert_surface_ann(record) for record in v0["surface_ann"]] + tables[SchemaName.KEYPOINT] = [self.convert_keypoint(record) for record in v0["keypoint"]] + # after the labels: an instance's count and first / last label are over all its labels + annotations = self._instance_annotations(tables) + tables[SchemaName.INSTANCE] = [ + self.convert_instance(record, annotations) for record in v0["instance"] + ] + return tables + + # --- core tables --- + + def convert_map(self, record: Record) -> Record | None: + """`sha256` from the map file, `projector` from its yaml; None if the map file is missing.""" + filename = record["filename"] or self.FALLBACKS["map_filename"] + filepath = osp.join(self.data_root, filename) + if not osp.exists(filepath): + return None + with open(filepath, "rb") as f: + sha256 = hashlib.sha256(f.read()).hexdigest() + fallbacks = self.FALLBACKS["map"] + return { + "token": record["token"], + "name": None, + "log_tokens": record["log_tokens"], + "map_id": fallbacks["map_id"], + "version_id": fallbacks["version_id"], + "source": fallbacks["source"], + "format": "lanelet2_osm", + "filename": filename, + "sha256": sha256, + "projector": self._read_projector(), + } + + def convert_vehicle(self, log: Record) -> Record: + """One vehicle per log, named after the 0.x `log.vehicle`, with placeholder dimensions.""" + fallbacks = self.FALLBACKS["vehicle"] + return { + "token": self._vehicle_token(log), + "name": self._none_if_empty(log["vehicle"]), + "model": fallbacks["model"], + "length": fallbacks["length"], + "width": fallbacks["width"], + "height": fallbacks["height"], + "wheel_base": fallbacks["wheel_base"], + "rear_axle_to_center": None, + "sensor_rig_id": None, + } + + def convert_log(self, record: Record, log2map: dict[str, str]) -> Record: + return { + "token": record["token"], + "logfile": self._none_if_empty(record["logfile"]), + "vehicle_token": self._vehicle_token(record), + "data_captured": record["data_captured"], + "location": self._none_if_empty(record["location"]), + "map_token": log2map.get(record["token"]), + } + + def convert_scene(self, record: Record) -> Record: + """Counts and first / last sample over the 1.0 samples (every frame).""" + samples = [sample for sample in self.samples if sample["scene_token"] == record["token"]] + return { + "token": record["token"], + "name": record["name"], + "description": self._none_if_empty(record["description"]), + "log_token": record["log_token"], + "nbr_samples": len(samples), + "first_sample_token": next(s["token"] for s in samples if s["prev"] is None), + "last_sample_token": next(s["token"] for s in samples if s["next"] is None), + } + + def convert_ego_pose(self, record: Record) -> Record: + fallbacks = self.FALLBACKS["ego_pose"] + return { + "token": record["token"], + "translation": record["translation"], + "rotation": record["rotation"], + "timestamp": record["timestamp"], + "twist": record.get("twist"), + "acceleration": record.get("acceleration"), + "geocoordinate": record.get("geocoordinate"), + "covariance": None, + "source": fallbacks["source"], + "frame": fallbacks["frame"], + } + + def convert_sensor(self, index: int, calibration: Record) -> Record: + """Merge 0.x `sensor` (per channel) and `calibrated_sensor` (per calibration). + + One row per calibration, keeping the `calibrated_sensor` token, indexed in file order. + """ + sensor = self.sensors[calibration["sensor_token"]] + modality = sensor["modality"] + return { + "token": calibration["token"], + "channel": sensor["channel"], + "index": index, + "modality": modality, + "translation": calibration["translation"], + "rotation": calibration["rotation"], + "camera": self.convert_camera(calibration) if modality == "camera" else None, + "lidar": self.convert_lidar(calibration) if modality == "lidar" else None, + "radar": None, + "hardware": None, + } + + def convert_camera(self, calibration: Record) -> Record: + """Size from `sample_data`, or from the first image if 0.x stored none.""" + distortion = calibration["camera_distortion"] + distortion_model = DistortionModel.from_num_coefficients(len(distortion)) + frames = self.frames[calibration["token"]] + sized = [record for record in frames if record.get("width") and record.get("height")] + if sized: + width, height = sized[0]["width"], sized[0]["height"] + else: + with Image.open(osp.join(self.data_root, frames[0]["filename"])) as image: + width, height = image.size + return { + "width": width, + "height": height, + "intrinsic": calibration["camera_intrinsic"], + "distortion": distortion, + "distortion_model": distortion_model.value, + "is_rectified": self.FALLBACKS["camera"]["is_rectified"], + "video": None, + } + + def convert_lidar(self, calibration: Record) -> Record: + """From the first frame: its format, its `.info` file (if any) and its header.""" + frames = self.frames[calibration["token"]] + first = frames[0] if frames else {} + info_filename = first.get("info_filename") + info = ( + load_json(osp.join(self.data_root, info_filename)) + if info_filename and osp.exists(osp.join(self.data_root, info_filename)) + else {} + ) + if first.get("fileformat") == "pcd": + filepath = osp.join(self.data_root, first["filename"]) + data = pypcd4.PointCloud.from_path(filepath).pc_data + container_format = "pcd" + fields = [{"name": name, "dtype": str(data.dtype[name])} for name in data.dtype.names] + else: + container_format = "bin" + num_pts_feats = info.get("num_pts_feats", 5) + fields = [{"name": name, "dtype": "float32"} for name in V0_BIN_FIELDS[:num_pts_feats]] + fallbacks = self.FALLBACKS["lidar"] + return { + "container": { + "format": container_format, + "path": None, + "frame_count": len(frames), + "compression": fallbacks["compression"], + }, + "fields": fields, + "source_channels": [ + self.sensors[source["sensor_token"]]["channel"] + for source in info.get("sources", []) + ], + "frame_id": fallbacks["frame_id"], + "is_undistorted": fallbacks["is_undistorted"], + } + + def convert_sample_data(self, record: Record) -> Record: + return { + "token": record["token"], + "sample_token": self.sample_of_frame[record["token"]], + "ego_pose_token": record["ego_pose_token"], + # 0.x calibrated_sensor rows become 1.0 sensor rows, with the same tokens. + "sensor_token": record["calibrated_sensor_token"], + "filename": record["filename"], + "frame_index": None, + "fileformat": record["fileformat"], + "num_points": None, + "timestamp": record["timestamp"], + "is_key_frame": self._is_key_frame(record), + "is_valid": record.get("is_valid", True), + "next": self._none_if_empty(record["next"]), + "prev": self._none_if_empty(record["prev"]), + } + + # --- label tables --- + + def convert_category(self, record: Record) -> Record: + """`is_thing` is false only for categories used by `surface_ann` (road, sky) and nothing else.""" + token = record["token"] + return { + "token": token, + "name": record["name"], + "index": record.get("index"), + "description": self._none_if_empty(record.get("description")), + "is_thing": token in self.thing_categories or token not in self.surface_categories, + "has_orientation": record.get("has_orientation", False), + "has_number": record.get("has_number", False), + } + + def convert_attribute(self, record: Record) -> Record: + return { + "token": record["token"], + "name": record["name"], + "description": self._none_if_empty(record.get("description")), + } + + def convert_visibilities(self) -> list[Record]: + """Levels as read by 0.x (aliases mapped); `unavailable` rows are dropped (visibility unknown).""" + rows = [] + for record in self.v0["visibility"]: + level = V0VisibilityLevel.from_value(record["level"]) + if level != V0VisibilityLevel.UNAVAILABLE: + rows.append( + { + "token": record["token"], + "level": level.value, + "description": self._none_if_empty(record.get("description")), + } + ) + return rows + + def convert_instance( + self, record: Record, annotations: dict[str, tuple[int, str, str]] + ) -> Record: + """`nbr_annotations` and the first / last annotation come from `annotations`: 1.0 counts + 3D boxes and 2D labels, and 0.x tools count them differently (some write 0 for 2D).""" + count, first, last = annotations.get(record["token"], (0, None, None)) + return { + "token": record["token"], + "name": record["instance_name"], + "index": None, + "category_token": record["category_token"], + "nbr_annotations": count, + "first_annotation_token": first, + "last_annotation_token": last, + "map_element": {"id": None, "type": None}, + } + + def convert_sample_annotation(self, record: Record, known_visibility: set[str]) -> Record: + """Visibility levels dropped by `convert_visibilities` become a null `visibility_token`.""" + visibility_token = record["visibility_token"] + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_token": self.sample_remap.get(record["sample_token"], record["sample_token"]), + "instance_token": record["instance_token"], + "attribute_tokens": record["attribute_tokens"], + "visibility_token": visibility_token if visibility_token in known_visibility else None, + "translation": record["translation"], + "size": record["size"], + "rotation": record["rotation"], + "velocity": record.get("velocity"), + "acceleration": record.get("acceleration"), + "num_lidar_pts": record["num_lidar_pts"], + "num_radar_pts": record["num_radar_pts"], + "next": self._none_if_empty(record["next"]), + "prev": self._none_if_empty(record["prev"]), + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + def convert_object_ann(self, record: Record) -> Record: + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_data_token": record["sample_data_token"], + "instance_token": record["instance_token"], + "category_token": record["category_token"], + "attribute_tokens": record["attribute_tokens"], + "bbox": record["bbox"], + "mask": record.get("mask"), + "orientation": record.get("orientation"), + "number": record.get("number"), + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + def convert_surface_ann(self, record: Record) -> Record: + """A 0.x semantic mask becomes an `object_ann` without instance; its box is the mask's.""" + mask = record["mask"] + rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} + x, y, w, h = (float(v) for v in cocomask.toBbox(rle)) + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_data_token": record["sample_data_token"], + "instance_token": None, + "category_token": record["category_token"], + "attribute_tokens": [], + "bbox": [x, y, x + w, y + h], + "mask": mask, + "orientation": None, + "number": None, + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + def convert_keypoint(self, record: Record) -> Record: + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_data_token": record["sample_data_token"], + "instance_token": record["instance_token"], + "category_tokens": record["category_tokens"], + "keypoints": record["keypoints"], + "num_keypoints": record["num_keypoints"], + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + # --- helpers --- + + def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: + """One 1.0 sample per 0.x frame: the frames of all channels with the same frame number. + + A frame with 0.x key samples keeps the earliest one's token; camera-only 0.x datasets have + one sample per camera image, merged here. Other frames get a new deterministic token. + + Returns: + The sample rows (chained by `next` / `prev` per scene), the sample token of each + `sample_data` token, and the token kept for each merged 0.x sample. + """ + v0_samples = {record["token"]: record for record in self.v0["sample"]} + annotated = {record["sample_token"] for record in self.v0["sample_annotation"]} + lidars = { + calibration["token"] + for calibration in self.v0["calibrated_sensor"] + if self.sensors[calibration["sensor_token"]]["modality"] == "lidar" + } + + rows: list[Record] = [] + sample_of_frame: dict[str, str] = {} + remap: dict[str, str] = {} + for frames in self._frame_groups().values(): + key_samples = sorted( + { + frame["sample_token"] + for frame in frames + if frame["is_key_frame"] and frame["sample_token"] in v0_samples + }, + key=lambda token: (v0_samples[token]["timestamp"], token), + ) + if key_samples: + token = key_samples[0] + remap.update({merged: token for merged in key_samples}) + else: + first = min(frame["token"] for frame in frames) + token = hashlib.md5(f"{first}/sample".encode()).hexdigest() + # 0.x non-key frames point to the next sample: same scene + linked = [ + frame["sample_token"] for frame in frames if frame["sample_token"] in v0_samples + ] + scene_token = ( + v0_samples[linked[0]]["scene_token"] if linked else self.v0["scene"][0]["token"] + ) + lidar_frames = [frame for frame in frames if frame["calibrated_sensor_token"] in lidars] + rows.append( + { + "token": token, + "timestamp": min(frame["timestamp"] for frame in lidar_frames or frames), + "scene_token": scene_token, + "has_key_frame": any(self._is_key_frame(frame) for frame in frames) + or bool(annotated & set(key_samples)), + "next": None, + "prev": None, + } + ) + sample_of_frame.update({frame["token"]: token for frame in frames}) + + by_scene: dict[str, list[Record]] = defaultdict(list) + for row in sorted(rows, key=lambda row: (row["timestamp"], row["token"])): + by_scene[row["scene_token"]].append(row) + for chain in by_scene.values(): + for row, following in zip(chain, chain[1:]): + row["next"], following["prev"] = following["token"], row["token"] + return [row for chain in by_scene.values() for row in chain], sample_of_frame, remap + + def _frame_groups(self) -> dict[int, list[Record]]: + """0.x `sample_data` grouped by frame number (`FRAME_NUMBER` in the file name), or by + position in each channel's frames in time if a file name has no number.""" + records = self.v0["sample_data"] + numbers = [self.FRAME_NUMBER.match(osp.basename(record["filename"])) for record in records] + groups: dict[int, list[Record]] = defaultdict(list) + if all(numbers): + for record, number in zip(records, numbers): + groups[int(number.group(1))].append(record) + else: + for frames in self.frames.values(): + for position, record in enumerate(sorted(frames, key=lambda r: r["timestamp"])): + groups[position].append(record) + return groups + + def _is_key_frame(self, record: Record) -> bool: + return record["is_key_frame"] or record["token"] in self.labeled_frames + + def _instance_annotations( + self, tables: dict[SchemaName, list[Record]] + ) -> dict[str, tuple[int, str, str]]: + """Return the number of labels and the first / last label token in time of each instance. + + A `SampleAnnotation` is at its sample's time, an `ObjectAnn` at its frame's time (as + sanity `REC013` checks). + """ + sample_times = { + sample["token"]: sample["timestamp"] for sample in tables[SchemaName.SAMPLE] + } + frame_times = { + frame["token"]: frame["timestamp"] for frame in tables[SchemaName.SAMPLE_DATA] + } + times: dict[str, list[tuple[int, str]]] = {} + for ann in tables[SchemaName.SAMPLE_ANNOTATION]: + times.setdefault(ann["instance_token"], []).append( + (sample_times[ann["sample_token"]], ann["token"]) + ) + for ann in tables[SchemaName.OBJECT_ANN]: + if ann["instance_token"] is not None: + times.setdefault(ann["instance_token"], []).append( + (frame_times[ann["sample_data_token"]], ann["token"]) + ) + return { + token: (len(entries), min(entries)[1], max(entries)[1]) + for token, entries in times.items() + } + + def _read_projector(self) -> Record: + """Read Autoware's `map_projector_info.yaml`, if any.""" + filepath = osp.join(self.data_root, self.FALLBACKS["map_projector_filename"]) + info = {} + if osp.exists(filepath): + with open(filepath) as f: + info = yaml.safe_load(f) or {} + origin = info.get("map_origin") or {} + return { + "type": info.get("projector_type"), + "mgrs_grid": info.get("mgrs_grid"), + "origin": { + "lat": origin.get("latitude"), + "lon": origin.get("longitude"), + "alt": origin.get("altitude"), + }, + } + + @staticmethod + def _vehicle_token(log: Record) -> str: + """Deterministic token: 0.x has no vehicle table.""" + return hashlib.md5(f"{log['token']}/vehicle".encode()).hexdigest() + + @staticmethod + def _none_if_empty(value: Any) -> Any: + """0.x writes "" for missing values; 1.0 uses None.""" + return None if value == "" else value + + @staticmethod + def _label_origin(record: Record) -> tuple[str, str | None, float | None]: + """`annotation_type`, `provenance` and `confidence` from the 0.x autolabel fields. + + Manual unless 0.x marks the label as automatic. `provenance` lists the autolabel models; + `confidence` is the model score when there is exactly one model. + """ + if not record.get("automatic_annotation", False): + return "manual", None, None + models = record.get("autolabel_metadata") or [] + provenance = ", ".join(model["name"] for model in models) or None + confidence = models[0]["score"] if len(models) == 1 else None + return "auto", provenance, confidence diff --git a/t4_devkit/schema/v1/loader.py b/t4_devkit/schema/v1/loader.py index 231d4ab3..39a92bfe 100644 --- a/t4_devkit/schema/v1/loader.py +++ b/t4_devkit/schema/v1/loader.py @@ -7,6 +7,7 @@ from t4_devkit.common.io import load_json +from .convert_v0 import V0Converter from .manifest import Manifest, load_manifest from .name import SchemaName from .registry import SCHEMAS @@ -24,12 +25,12 @@ class T4Tables: Attributes: data_root (str): Dataset root. - manifest (Manifest): The dataset manifest. + manifest (Manifest | None): The dataset manifest, or None for a 0.x dataset. tables (dict[SchemaName, list[SchemaTable]]): Records per table; empty for an absent optional table. """ data_root: str - manifest: Manifest + manifest: Manifest | None tables: dict[SchemaName, list[SchemaTable]] _token2idx: dict[SchemaName, dict[str, int]] = field(init=False) @@ -86,7 +87,8 @@ def _fill_shortcuts(self) -> None: def load_dataset(data_root: PathLike) -> T4Tables: """Load a dataset as 1.x records. - Read the raw table dicts -> build and validate the records once. + Read the raw table dicts (0.x: converted to 1.0 dicts, best effort) -> build and validate + the records once. Args: data_root (PathLike): Dataset root, i.e. the `/` directory. @@ -95,15 +97,12 @@ def load_dataset(data_root: PathLike) -> T4Tables: The loaded tables. Raises: - NotImplementedError: If the dataset is 0.x (no `manifest.json`). FileNotFoundError: If `manifest.json` does not list a required table. """ manifest = load_manifest(data_root) - if manifest is None: - # 0.x: converted to 1.x dicts here in a later step. - raise NotImplementedError("Loading 0.x datasets is not supported yet.") - - raw_tables = _read_tables(data_root, manifest) + raw_tables = ( + V0Converter(data_root).convert() if manifest is None else _read_tables(data_root, manifest) + ) tables = { name: [SCHEMAS.get(name).from_dict(data) for data in records] for name, records in raw_tables.items() diff --git a/t4_devkit/schema/v1/sample_annotation.py b/t4_devkit/schema/v1/sample_annotation.py index c4a21489..021b966a 100644 --- a/t4_devkit/schema/v1/sample_annotation.py +++ b/t4_devkit/schema/v1/sample_annotation.py @@ -25,7 +25,7 @@ class SampleAnnotation(SchemaBase): sample_token (str): Foreign key pointing to the sample. instance_token (str): Foreign key pointing to the instance. attribute_tokens (list[str]): Foreign keys pointing to the attributes. - visibility_token (str): Foreign key pointing to the visibility level. + visibility_token (str | None): Foreign key pointing to the visibility level, or None if unknown. translation (Vector3): Box center [x, y, z] in [m]. size (Vector3): Box size [width, length, height] in [m]. rotation (Quaternion): Box orientation as quaternion [w, x, y, z]. @@ -52,7 +52,11 @@ class SampleAnnotation(SchemaBase): validators.instance_of(list), ) ) - visibility_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + visibility_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) translation: Vector3 = field(converter=Vector3) size: Vector3 = field(converter=Vector3) rotation: Quaternion = field(converter=to_quaternion) diff --git a/t4_devkit/schema/v1/sensor.py b/t4_devkit/schema/v1/sensor.py index a20bec90..5cd96b82 100644 --- a/t4_devkit/schema/v1/sensor.py +++ b/t4_devkit/schema/v1/sensor.py @@ -41,6 +41,14 @@ def num_coefficients(self) -> int: """Number of distortion coefficients of this model.""" return {"plumb_bob": 5, "rational_polynomial": 8, "equidistant": 4, "none": 0}[self.value] + @classmethod + def from_num_coefficients(cls, num_coefficients: int) -> DistortionModel: + """The model with this number of coefficients (each model has a different count).""" + for model in cls: + if model.num_coefficients == num_coefficients: + return model + raise ValueError(f"No distortion model has {num_coefficients} coefficients") + @unique class VideoFormat(str, Enum): diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py new file mode 100644 index 00000000..6c3b82ee --- /dev/null +++ b/tests/schema/v1/test_convert_v0.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +from t4_devkit.schema.v1 import SchemaName, V0Converter, load_dataset + +SAMPLE_V0 = Path(__file__).parents[2] / "sample" / "t4dataset" + + +def test_load_v0_dataset() -> None: + """A 0.x dataset loads as 1.0 records, with placeholders for what 0.x never stored.""" + dataset = load_dataset(SAMPLE_V0) + tables = dataset.tables + + assert dataset.manifest is None + assert len(tables[SchemaName.SAMPLE]) == 3 + assert len(tables[SchemaName.SAMPLE_DATA]) == 9 + + # calibrated_sensor rows become sensor rows with the same tokens + sensors = {sensor.channel: sensor for sensor in tables[SchemaName.SENSOR]} + assert sorted(sensor.index for sensor in sensors.values()) == [0, 1, 2] + camera = sensors["CAM_FRONT"].camera + assert (camera.width, camera.height) == (800, 600) + assert camera.distortion_model == "plumb_bob" + lidar = sensors["LIDAR_TOP"].lidar + assert lidar.container.format == "bin" + assert lidar.container.frame_count == 3 + assert [field.name for field in lidar.fields] == ["x", "y", "z", "intensity", "channel"] + for record in tables[SchemaName.SAMPLE_DATA]: + assert dataset.get(SchemaName.SENSOR, record.sensor_token).channel in sensors + assert record.frame_index is None + + # log -> vehicle (placeholders) and map (hash from the file) + log = tables[SchemaName.LOG][0] + vehicle = dataset.get(SchemaName.VEHICLE, log.vehicle_token) + assert vehicle.name == "test_vehicle" + assert vehicle.model == V0Converter.FALLBACKS["vehicle"]["model"] + map_ = dataset.get(SchemaName.MAP, log.map_token) + assert map_.map_id == V0Converter.FALLBACKS["map"]["map_id"] + assert len(map_.sha256) == 64 + + +def test_load_v0_labels() -> None: + """Labels convert; surface_ann becomes object_ann without instance, boxed by its mask.""" + dataset = load_dataset(SAMPLE_V0) + tables = dataset.tables + + assert len(tables[SchemaName.SAMPLE_ANNOTATION]) == 4 + assert all(r.annotation_type == "manual" for r in tables[SchemaName.SAMPLE_ANNOTATION]) + assert tables[SchemaName.INSTANCE][0].name == "car_001" + # 1.0 counts 3D boxes and 2D labels (0.x tools count them differently) + for instance in tables[SchemaName.INSTANCE]: + labels = [ + r + for table in (SchemaName.SAMPLE_ANNOTATION, SchemaName.OBJECT_ANN) + for r in tables[table] + if r.instance_token == instance.token + ] + assert instance.nbr_annotations == len(labels) + + surfaces = [r for r in tables[SchemaName.OBJECT_ANN] if r.instance_token is None] + assert len(surfaces) == 2 + for record in surfaces: + category = dataset.get(SchemaName.CATEGORY, record.category_token) + assert not category.is_thing + xmin, ymin, xmax, ymax = record.bbox + assert 0 <= xmin < xmax <= record.mask.size[1] and 0 <= ymin < ymax <= record.mask.size[0] + + +def test_load_v0_autolabel(tmp_path: Path) -> None: + """A 0.x autolabel becomes annotation_type auto, with the model as provenance.""" + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + path = data_root / "annotation" / "sample_annotation.json" + records = json.loads(path.read_text()) + records[0]["automatic_annotation"] = True + records[0]["autolabel_metadata"] = [{"name": "centerpoint-v2", "score": 0.8}] + path.write_text(json.dumps(records)) + + annotation = load_dataset(data_root).tables[SchemaName.SAMPLE_ANNOTATION][0] + assert annotation.annotation_type == "auto" + assert annotation.provenance == "centerpoint-v2" + assert annotation.confidence == 0.8 + + +def test_load_v0_visibility(tmp_path: Path) -> None: + """0.x level aliases are mapped; `unavailable` is dropped and becomes a null visibility_token.""" + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + path = data_root / "annotation" / "visibility.json" + records = json.loads(path.read_text()) + records[0]["level"] = "v80-100" + records[1]["level"] = "unavailable" + path.write_text(json.dumps(records)) + path = data_root / "annotation" / "sample_annotation.json" + annotations = json.loads(path.read_text()) + annotations[0]["visibility_token"] = records[1]["token"] + path.write_text(json.dumps(annotations)) + + tables = load_dataset(data_root).tables + assert [r.level for r in tables[SchemaName.VISIBILITY]] == ["full", "partial", "none"] + assert tables[SchemaName.SAMPLE_ANNOTATION][0].visibility_token is None + + +def _copy_v0(tmp_path: Path) -> Path: + data_root = tmp_path / "dataset" + shutil.copytree(SAMPLE_V0, data_root) + return data_root + + +def _edit_v0(data_root: Path, name: str, change) -> None: + path = data_root / "annotation" / f"{name}.json" + path.write_text(json.dumps(change(json.loads(path.read_text())))) + + +def test_load_v0_frames(tmp_path: Path) -> None: + """Every 0.x frame is a 1.0 sample, grouped by frame number; 0.x key samples keep their token.""" + data_root = _copy_v0(tmp_path) + sample_data = json.loads((data_root / "annotation" / "sample_data.json").read_text()) + lidar2 = next(r for r in sample_data if r["filename"] == "data/LIDAR_TOP/002.pcd.bin") + lidar3 = next(r for r in sample_data if r["filename"] == "data/LIDAR_TOP/003.pcd.bin") + + def sweep(records: list[dict]) -> list[dict]: + # as the 0.x converter writes a non-key frame: linked to the next sample + for record in records: + if record["token"] == lidar2["token"]: + record.update(is_key_frame=False, sample_token=lidar3["sample_token"]) + return records + + _edit_v0(data_root, "sample_data", sweep) + dataset = load_dataset(data_root) + frame = dataset.get(SchemaName.SAMPLE_DATA, lidar2["token"]) + assert frame.sample_token == lidar2["sample_token"] # its own frame's sample, not the next + assert not frame.is_key_frame + assert dataset.get(SchemaName.SAMPLE, frame.sample_token).has_key_frame # cameras are key + samples = dataset.tables[SchemaName.SAMPLE] + assert len(samples) == 3 and all(len(sample.data) == 3 for sample in samples) + + +def test_load_v0_per_camera_samples(tmp_path: Path) -> None: + """0.x samples of one frame (one per camera image) merge into one 1.0 sample.""" + data_root = _copy_v0(tmp_path) + samples = json.loads((data_root / "annotation" / "sample.json").read_text()) + first = samples[0]["token"] + extra = {**samples[0], "token": "f" * 32} # sorts after `first` + _edit_v0(data_root, "sample", lambda records: [*records, extra]) + _edit_v0( + data_root, + "sample_data", + lambda records: [ + {**r, "sample_token": extra["token"]} if r["filename"] == "data/CAM_BACK/001.jpg" else r + for r in records + ], + ) + + dataset = load_dataset(data_root) + assert len(dataset.tables[SchemaName.SAMPLE]) == 3 + assert set(dataset.get(SchemaName.SAMPLE, first).data) == {"LIDAR_TOP", "CAM_FRONT", "CAM_BACK"} + + +def test_load_v0_mask_only_frame_is_key(tmp_path: Path) -> None: + """A non-key 0.x frame whose only label is a semantic mask (`surface_ann`) is a key frame.""" + data_root = _copy_v0(tmp_path) + token = json.loads((data_root / "annotation" / "surface_ann.json").read_text())[0][ + "sample_data_token" + ] + _edit_v0( + data_root, + "sample_data", + lambda records: [ + dict(r, is_key_frame=False) if r["token"] == token else r for r in records + ], + ) + _edit_v0( + data_root, + "object_ann", + lambda records: [r for r in records if r["sample_data_token"] != token], + ) + assert load_dataset(data_root).get(SchemaName.SAMPLE_DATA, token).is_key_frame + + +def test_load_v0_labeled_frame_is_key(tmp_path: Path) -> None: + """A 0.x 2D label on a non-key frame makes it a key frame.""" + data_root = _copy_v0(tmp_path) + labeled = json.loads((data_root / "annotation" / "object_ann.json").read_text())[0] + _edit_v0( + data_root, + "sample_data", + lambda records: [ + {**r, "is_key_frame": False} if r["token"] == labeled["sample_data_token"] else r + for r in records + ], + ) + frame = load_dataset(data_root).get(SchemaName.SAMPLE_DATA, labeled["sample_data_token"]) + assert frame.is_key_frame diff --git a/tests/schema/v1/test_loader.py b/tests/schema/v1/test_loader.py index 00a1767b..6de940e3 100644 --- a/tests/schema/v1/test_loader.py +++ b/tests/schema/v1/test_loader.py @@ -56,11 +56,6 @@ def test_shortcuts() -> None: assert {r.category_name for r in dataset.tables[SchemaName.OBJECT_ANN]} == {"car", "road"} -def test_v0_not_supported_yet() -> None: - with pytest.raises(NotImplementedError): - load_dataset(SAMPLE_ROOT / "t4dataset") - - def test_missing_required_table(tmp_path: Path) -> None: with pytest.raises(FileNotFoundError, match="vehicle"): load_dataset(_copy_without_table(tmp_path, "vehicle"))