diff --git a/docs/cli/t4viz.md b/docs/cli/t4viz.md index fc953d8c..d046621d 100644 --- a/docs/cli/t4viz.md +++ b/docs/cli/t4viz.md @@ -58,7 +58,7 @@ t4viz instance ... [OPTIONS] This command performs the same behavior with [`T4Devkit.render_pointcloud(...)`](../tutorials/render.md#rendering-pointcloud). -If a pointcloud `sample_data` record has `info_filename`, `t4viz pointcloud` loads the metainfo JSON automatically and uses `num_pts_feats` to parse extended point layouts. Without that file, it falls back to the standard 5-feature layout. +The point layout comes from the file (Parquet, BIN, PCD) or from `sensor.lidar.fields` (`.bin`; for 0.x datasets it is taken from the `.info` files when loading). For options, run `t4viz pointcloud -h`. diff --git a/docs/tutorials/customize.md b/docs/tutorials/customize.md index 8b36e4e9..b56e8782 100644 --- a/docs/tutorials/customize.md +++ b/docs/tutorials/customize.md @@ -57,8 +57,8 @@ from __future__ import annotations from attrs import define, field -from t4_devkit.schema import SCHEMAS, SchemaName, SchemaBase -from t4_devkit.common import load_json +from t4_devkit.schema import SchemaBase +from t4_devkit.schema.v1 import SCHEMAS, SchemaName __all__ = ["CustomAttribute"] @@ -77,6 +77,8 @@ class CustomAttribute(SchemaBase): description: str | None = field(default=None) ``` +`T4Devkit` builds every table with the T4Dataset 1.0 registry (`t4_devkit.schema.v1.SCHEMAS`), for 1.0 datasets and for 0.x datasets (converted to 1.0 records when loading), so a custom class receives 1.0 records. + Note that `CustomAttribute` should be imported before instantiating `T4Devkit` class. Then modify `__init__.py` in order to import it automatically: diff --git a/docs/tutorials/initialize.md b/docs/tutorials/initialize.md index 96822783..b289a6dc 100644 --- a/docs/tutorials/initialize.md +++ b/docs/tutorials/initialize.md @@ -47,30 +47,33 @@ You can initialize a `T4Devkit` instance as follows: >>> t4 = T4Devkit("data/tier4/", verbose=True) ====== -Loading T4 tables in `annotation`... -Reverse indexing... -Done reverse indexing in 0.007 seconds. -====== -13 attribute -7 calibrated_sensor -8 category -2524 ego_pose -106 instance -1 log -1 map +Loading T4 tables... 72 sample -2390 sample_annotation 2524 sample_data -4 visibility +2524 ego_pose 7 sensor 1 scene +1 log +1 vehicle +1 map +0 vehicle_state +8 category +13 attribute +4 visibility +106 instance +2390 sample_annotation 1326 object_ann -0 surface_ann 0 keypoint +0 traffic_light +0 route +0 scenario +0 caption Done loading in 0.061 seconds. ====== ``` +Both T4Dataset 1.0 datasets (with `manifest.json`) and 0.x datasets load, as 1.0 records: 0.x tables are converted when loading (see [Loading 0.x Datasets](../schema/v1/index.md#loading-0x-datasets)). + Note that if you doesn't specify `revision` parameter in construction, it searches the latest version of the dataset. By specifying `revision=`, you can load the specific version of the dataset. @@ -90,7 +93,7 @@ By specifying `revision=`, you can load the specific version of the dat ### `sample` -`sample` is an annotated keyframe of a scene at a given timestamp. +`sample` is one frame of a scene: the observations of all channels at that moment. Every frame is a sample; `sample.has_key_frame` marks the annotated ones (key frames), which carry the labels. ```python >>> first_sample_token = my_scene.first_sample_token @@ -184,22 +187,12 @@ An `attribute` is a property of an instance that may change throughout different ### `sensor` T4 dataset consists of several type of sensors. -The supported sensor modalities and channels are defined in `t4_devkit/schema/tables/sensor.py`. +Each `sensor` row is one sensor with its calibration: `translation` and `rotation` with respect to the ego vehicle body frame, and a block for its modality (`camera` with intrinsics and image size, or `lidar` with its point fields). ```python >>> t4.sensor ``` -### `calibrated_sensor` - -`calibrated_sensor` consists of the definition of a calibration of a particular sensor based on a vehicle. - -```python ->>> t4.calibrated_sensor -``` - -Note that the `translation` and `rotation` parameters are given with respect to the ego vehicle body frame. - ### `ego_pose` `ego_pose` contains information about the `translation` and `rotation` of the ego vehicle, with respect to the global coordinate system. diff --git a/docs/tutorials/render.md b/docs/tutorials/render.md index c6286115..61c09cc4 100644 --- a/docs/tutorials/render.md +++ b/docs/tutorials/render.md @@ -36,7 +36,9 @@ If you want to visualize annotation results, `T4Devkit` supports some rendering >>> t4.render_pointcloud() ``` -If `SampleData.info_filename` points to a pointcloud metainfo JSON file, `T4Devkit.render_pointcloud()` loads it automatically. This allows rendering pointclouds with extended per-point features such as `return_type` or `timestamp`. When no metainfo file is available, the standard 5-feature layout is assumed. +Frames are read by their own format: point layouts come from the file (Parquet, PCD) or from `sensor.lidar.fields` (`.bin`; for 0.x datasets it is taken from the `.info` files when loading), and camera frames from per-frame images or from the channel's AV1 video (`sample_data.frame_index`). + +0.x LiDAR segmentation (`lidarseg`) is not rendered by `T4Devkit`. ![Render PointCloud GIF](../assets/render_pointcloud.gif) @@ -129,11 +131,11 @@ from t4_devkit.dataclass import LidarPointCloud from t4_devkit.viewer import PointCloudColorMode # Point cloud channel name >>> lidar_channel = "LIDAR_TOP" -# Load point cloud from file. -# Pass metainfo_filepath when the binary stores more than 5 float32 values per point. +# Load point cloud from file: one frame of a Parquet file, or a .pcd / .bin file. +# A .bin file has no header: pass its layout, e.g. from `sensor.lidar.fields`. >>> pointcloud = LidarPointCloud.from_file( -... , -... metainfo_filepath=, +... , +... frame_index=, ... ) >>> color_mode = PointCloudColorMode.DISTANCE >>> viewer.render_pointcloud(seconds, lidar_channel, pointcloud, color_mode) diff --git a/t4_devkit/helper/rendering.py b/t4_devkit/helper/rendering.py index ec3147d0..123b1378 100644 --- a/t4_devkit/helper/rendering.py +++ b/t4_devkit/helper/rendering.py @@ -11,12 +11,12 @@ import numpy as np import yaml +from t4_devkit.common.io import load_image from t4_devkit.common.timestamp import microseconds2seconds, seconds2microseconds -from t4_devkit.dataclass import LidarPointCloud, RadarPointCloud, SegmentationPointCloud +from t4_devkit.dataclass import LidarPointCloud, RadarPointCloud from t4_devkit.schema import SensorModality from t4_devkit.viewer import ( EntityPath, - PointCloudColorMode, RerunViewer, ViewerBuilder, format_entity, @@ -24,17 +24,14 @@ if TYPE_CHECKING: from t4_devkit import T4Devkit - from t4_devkit.schema import ( - CalibratedSensor, + from t4_devkit.schema.v1 import ( EgoPose, Instance, ObjectAnn, Sample, - SampleAnnotation, SampleData, Scene, Sensor, - SurfaceAnn, ) __all__ = ["RenderingHelper"] @@ -58,22 +55,32 @@ def __init__(self, t4: T4Devkit) -> None: t4 (T4Devkit): `T4Devkit` instance. """ self._t4 = t4 + # Class IDs for colors: `category.index`, or the list position when it is not set + # (1.0 sets it only for datasets with point-wise labels). self._label2id: dict[str, int] = { - category.name: category.index for category in self._t4.category - } - self._sample_data_to_lidarseg_filename: dict[str, str] = { - lidarseg.sample_data_token: lidarseg.filename for lidarseg in self._t4.lidarseg + category.name: position if category.index is None else category.index + for position, category in enumerate(self._t4.category) } self._executor = concurrent.futures.ThreadPoolExecutor() - def _has_lidarseg(self) -> bool: - """Return `True` if the dataset includes lidarseg.""" - return bool(self._sample_data_to_lidarseg_filename) + def _first_sample_data_tokens(self, modality: SensorModality) -> list[str]: + """The first frame of each channel of a modality, in sensor order. - def _find_lidarseg_file(self, sample_data_token: str) -> str | None: - """Try to find lidarseg filename from the sample data token.""" - return self._sample_data_to_lidarseg_filename.get(sample_data_token) + A recalibration adds a `sensor` row with the same channel, and source LiDARs of a merged + channel have no frames, so channels are taken once and sensors without frames skipped. + """ + first_tokens: dict[str, str] = {} + for sensor in self._t4.sensor: + if sensor.modality != modality or sensor.first_sd_token is None: + continue + current = first_tokens.get(sensor.channel) + if current is None or ( + self._t4.get("sample_data", sensor.first_sd_token).timestamp + < self._t4.get("sample_data", current).timestamp + ): + first_tokens[sensor.channel] = sensor.first_sd_token + return list(first_tokens.values()) def _init_viewer( self, @@ -91,9 +98,13 @@ def _init_viewer( render_ann (bool, optional): Indicates whether to render annotations. save_dir (str | None, optional): Directory path to save the rendering record. """ - cameras = [ - sensor.channel for sensor in self._t4.sensor if sensor.modality == SensorModality.CAMERA - ] + cameras = list( + dict.fromkeys( + sensor.channel + for sensor in self._t4.sensor + if sensor.modality == SensorModality.CAMERA + ) + ) builder = ( ViewerBuilder().with_spatial3d().with_spatial2d(cameras=cameras, contents=contents) @@ -130,7 +141,7 @@ def _load_contents(self, mode: RenderingMode, entity_child: str = "") -> list[st case RenderingMode.SCENE | RenderingMode.INSTANCE: # project 3D boxes/velocities/futures on image if there is no 2D annotation entity_root = format_entity(EntityPath.MAP, entity_child) - if len(self._t4.object_ann) == 0 and len(self._t4.surface_ann) == 0: + if len(self._t4.object_ann) == 0: contents = [ format_entity(entity_root, EntityPath.BOX), format_entity(entity_root, EntityPath.VELOCITY), @@ -161,21 +172,9 @@ def render_scene( show_map (bool, optional): Whether to render the map. """ # search first sample data tokens - first_lidar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.LIDAR - ] - first_radar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.RADAR - ] - first_camera_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.CAMERA - ] + first_lidar_tokens = self._first_sample_data_tokens(SensorModality.LIDAR) + first_radar_tokens = self._first_sample_data_tokens(SensorModality.RADAR) + first_camera_tokens = self._first_sample_data_tokens(SensorModality.CAMERA) app_id = f"scene@{self._t4.dataset_id}" contents = self._load_contents(RenderingMode.SCENE) @@ -188,18 +187,11 @@ def render_scene( first_sample: Sample = self._t4.get("sample", scene.first_sample_token) max_timestamp_us = first_sample.timestamp + seconds2microseconds(max_time_seconds) - pointcloud_color_mode = ( - PointCloudColorMode.SEGMENTATION - if self._has_lidarseg() - else PointCloudColorMode.DISTANCE - ) - futures = ( self._render_lidar_and_ego( viewer=viewer, first_lidar_tokens=first_lidar_tokens, max_timestamp_us=max_timestamp_us, - color_mode=pointcloud_color_mode, ) + self._render_radars( viewer=viewer, @@ -253,18 +245,16 @@ def render_instance( last_sample: Sample | None = None for token in instance_tokens: instance: Instance = self._t4.get("instance", token) - first_ann: SampleAnnotation = self._t4.get( - "sample_annotation", instance.first_annotation_token - ) - current_first_sample: Sample = self._t4.get("sample", first_ann.sample_token) - if first_sample is None or current_first_sample.timestamp < first_sample.timestamp: + current_first_sample = self._annotation_sample(instance.first_annotation_token) + if current_first_sample is not None and ( + first_sample is None or current_first_sample.timestamp < first_sample.timestamp + ): first_sample = current_first_sample - last_ann: SampleAnnotation = self._t4.get( - "sample_annotation", instance.last_annotation_token - ) - current_last_sample: Sample = self._t4.get("sample", last_ann.sample_token) - if last_sample is None or current_last_sample.timestamp > last_sample.timestamp: + current_last_sample = self._annotation_sample(instance.last_annotation_token) + if current_last_sample is not None and ( + last_sample is None or current_last_sample.timestamp > last_sample.timestamp + ): last_sample = current_last_sample if first_sample is None or last_sample is None: @@ -277,21 +267,9 @@ def render_instance( max_timestamp_us = last_sample.timestamp # search first sample data tokens - first_lidar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.LIDAR - ] - first_radar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.RADAR - ] - first_camera_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.CAMERA - ] + first_lidar_tokens = self._first_sample_data_tokens(SensorModality.LIDAR) + first_radar_tokens = self._first_sample_data_tokens(SensorModality.RADAR) + first_camera_tokens = self._first_sample_data_tokens(SensorModality.CAMERA) app_id = f"instance@{self._t4.dataset_id}" contents = self._load_contents(RenderingMode.INSTANCE) @@ -300,18 +278,11 @@ def render_instance( if show_map: self._try_render_map(viewer) - pointcloud_color_mode = ( - PointCloudColorMode.SEGMENTATION - if self._has_lidarseg() - else PointCloudColorMode.DISTANCE - ) - futures = ( self._render_lidar_and_ego( viewer=viewer, first_lidar_tokens=first_lidar_tokens, max_timestamp_us=max_timestamp_us, - color_mode=pointcloud_color_mode, ) + self._render_radars( viewer=viewer, @@ -363,16 +334,12 @@ def render_pointcloud( Add an option of rendering radar channels. """ # search first lidar sample data token - first_lidar_token: str | None = None - first_camera_tokens: list[str] = [] - for sensor in self._t4.sensor: - if sensor.modality == SensorModality.LIDAR: - first_lidar_token = sensor.first_sd_token - elif sensor.modality == SensorModality.CAMERA: - first_camera_tokens.append(sensor.first_sd_token) + first_lidar_tokens = self._first_sample_data_tokens(SensorModality.LIDAR) + first_camera_tokens = self._first_sample_data_tokens(SensorModality.CAMERA) - if first_lidar_token is None: + if not first_lidar_tokens: raise ValueError("There is no 3D pointcloud data.") + first_lidar_token = first_lidar_tokens[0] first_lidar_sample_data: SampleData = self._t4.get("sample_data", first_lidar_token) max_timestamp_us = first_lidar_sample_data.timestamp + seconds2microseconds( @@ -410,30 +377,20 @@ def _try_render_map(self, viewer: RerunViewer) -> None: def _render_sensor_calibration(self, viewer: RerunViewer, sample_data_token: str) -> None: sample_data: SampleData = self._t4.get("sample_data", sample_data_token) - calibration: CalibratedSensor = self._t4.get( - "calibrated_sensor", sample_data.calibrated_sensor_token - ) - sensor: Sensor = self._t4.get("sensor", calibration.sensor_token) - resolution = ( - (sample_data.width, sample_data.height) - if sensor.modality == SensorModality.CAMERA - else None - ) - viewer.render_calibration(sensor=sensor, calibration=calibration, resolution=resolution) + sensor: Sensor = self._t4.get("sensor", sample_data.sensor_token) + viewer.render_calibration(sensor) def _render_lidar_and_ego( self, viewer: RerunViewer, first_lidar_tokens: list[str], max_timestamp_us: float, - *, - color_mode: PointCloudColorMode = PointCloudColorMode.DISTANCE, ) -> list[Future]: def _render_single_lidar(first_lidar_token: str) -> None: self._render_sensor_calibration(viewer=viewer, sample_data_token=first_lidar_token) current_lidar_token = first_lidar_token - while current_lidar_token != "": + while current_lidar_token is not None: sample_data: SampleData = self._t4.get("sample_data", current_lidar_token) current_lidar_token = sample_data.next @@ -442,34 +399,19 @@ def _render_single_lidar(first_lidar_token: str) -> None: ego_pose: EgoPose = self._t4.get("ego_pose", sample_data.ego_pose_token) viewer.render_ego(ego_pose=ego_pose) - metainfo_filepath = ( - osp.join(self._t4.data_root, sample_data.info_filename) - if sample_data.info_filename - else None - ) - # render segmentation pointcloud if available, otherwise render raw pointcloud - if color_mode == PointCloudColorMode.SEGMENTATION: - label_filename = self._find_lidarseg_file(sample_data.token) - if label_filename is None: - continue - - pointcloud = SegmentationPointCloud.from_file( - point_filepath=osp.join(self._t4.data_root, sample_data.filename), - label_filepath=osp.join(self._t4.data_root, label_filename), - metainfo_filepath=metainfo_filepath, - ) - else: - pointcloud = LidarPointCloud.from_file( - osp.join(self._t4.data_root, sample_data.filename), - metainfo_filepath=metainfo_filepath, - ) + # the layout comes from the file, or from the sensor row for `.bin` + sensor: Sensor = self._t4.get("sensor", sample_data.sensor_token) + pointcloud = LidarPointCloud.from_file( + osp.join(self._t4.data_root, sample_data.filename), + frame_index=sample_data.frame_index, + fields=[(field.name, field.dtype) for field in sensor.lidar.fields], + ) viewer.render_pointcloud( seconds=microseconds2seconds(sample_data.timestamp), channel=sample_data.channel, pointcloud=pointcloud, - color_mode=color_mode, ) return [self._executor.submit(_render_single_lidar, token) for token in first_lidar_tokens] @@ -484,7 +426,7 @@ def _render_single_radar(first_radar_token: str) -> None: self._render_sensor_calibration(viewer=viewer, sample_data_token=first_radar_token) current_radar_token = first_radar_token - while current_radar_token != "": + while current_radar_token is not None: sample_data: SampleData = self._t4.get("sample_data", current_radar_token) if max_timestamp_us < sample_data.timestamp: @@ -513,16 +455,23 @@ def _render_single_camera(first_camera_token: str) -> None: self._render_sensor_calibration(viewer=viewer, sample_data_token=first_camera_token) current_camera_token = first_camera_token - while current_camera_token != "": + while current_camera_token is not None: sample_data: SampleData = self._t4.get("sample_data", current_camera_token) if max_timestamp_us < sample_data.timestamp: break + # An image file is passed as is (sent encoded); a video frame is decoded. + filepath = osp.join(self._t4.data_root, sample_data.filename) + image = ( + filepath + if sample_data.frame_index is None + else load_image(filepath, frame_index=sample_data.frame_index) + ) viewer.render_image( seconds=microseconds2seconds(sample_data.timestamp), camera=sample_data.channel, - image=osp.join(self._t4.data_root, sample_data.filename), + image=image, ) current_camera_token = sample_data.next @@ -541,16 +490,21 @@ def _render_annotation3ds( instance_tokens: Sequence[str] | None = None, ) -> None: current_sample_token = first_sample_token - while current_sample_token != "": + while current_sample_token is not None: sample: Sample = self._t4.get("sample", current_sample_token) if max_timestamp_us < sample.timestamp: break + # labels are on key frames only; logging none in between would clear them + if not sample.has_key_frame: + current_sample_token = sample.next + continue + if instance_tokens is not None: boxes = [] for ann_token in sample.ann_3ds: - ann: SampleAnnotation = self._t4.get("sample_annotation", ann_token) + ann = self._t4.get("sample_annotation", ann_token) if ann.instance_token in instance_tokens: boxes.append(self._t4.get_box3d(ann_token, future_seconds=future_seconds)) else: @@ -571,18 +525,23 @@ def _render_annotation2ds( instance_tokens: Sequence[str] | None = None, ) -> None: current_sample_token = first_sample_token - while current_sample_token != "": + while current_sample_token is not None: sample: Sample = self._t4.get("sample", current_sample_token) if max_timestamp_us < sample.timestamp: break + # labels are on key frames only; logging none in between would clear them + if not sample.has_key_frame: + current_sample_token = sample.next + continue + boxes = [] # For segmentation masks # TODO: declare specific class for segmentation mask in `dataclass` camera_masks: dict[str, dict[str, list]] = {} - # Object Annotation + # Object Annotation, including regions without an instance (road, sky) for obj_ann_token in sample.ann_2ds: obj_ann: ObjectAnn = self._t4.get("object_ann", obj_ann_token) box = self._t4.get_box2d(obj_ann_token) @@ -612,20 +571,6 @@ def _render_annotation2ds( # Render 2D box viewer.render_box2ds(microseconds2seconds(sample.timestamp), boxes) - if instance_tokens is None: - # Surface Annotation - for ann_token in sample.surface_anns: - surface_ann: SurfaceAnn = self._t4.get("surface_ann", ann_token) - sample_data: SampleData = self._t4.get( - "sample_data", surface_ann.sample_data_token - ) - camera_masks = _append_mask( - camera_masks, - camera=sample_data.channel, - ann=surface_ann, - class_id=self._label2id[surface_ann.category_name], - ) - # Render 2D segmentation image for camera, data in camera_masks.items(): viewer.render_segmentation2d( @@ -635,15 +580,30 @@ def _render_annotation2ds( # TODO: add support of rendering keypoints current_sample_token = sample.next + def _annotation_sample(self, annotation_token: str | None) -> Sample | None: + """The sample of an instance's first or last annotation, or None if there is none. + + The annotation is a `sample_annotation`, or an `object_ann` for instances labeled only in + 2D, whose sample is that of its frame. + """ + if annotation_token is None: + return None + try: + sample_token = self._t4.get("sample_annotation", annotation_token).sample_token + except KeyError: + object_ann: ObjectAnn = self._t4.get("object_ann", annotation_token) + sample_token = self._t4.get("sample_data", object_ann.sample_data_token).sample_token + return None if sample_token is None else self._t4.get("sample", sample_token) + def _append_mask( camera_masks: dict[str, dict[str, list]], camera: str, - ann: ObjectAnn | SurfaceAnn, + ann: ObjectAnn, class_id: int, uuid: str | None = None, ) -> dict[str, dict[str, list]]: - """Append segmentation mask data from `ObjectAnn/SurfaceAnn`. + """Append segmentation mask data from `ObjectAnn`. TODO: This function should be removed after declaring specific dataclass for 2D segmentation. @@ -651,7 +611,7 @@ def _append_mask( Args: camera_masks (dict[str, dict[str, list]]): Key-value data mapping camera name and mask data. camera (str): Name of camera channel. - ann (ObjectAnn | SurfaceAnn): Annotation object. + ann (ObjectAnn): Annotation object. class_id (int): Class ID. uuid (str | None, optional): Unique instance identifier. diff --git a/t4_devkit/helper/timeseries.py b/t4_devkit/helper/timeseries.py index 0559be4e..c4d1fe32 100644 --- a/t4_devkit/helper/timeseries.py +++ b/t4_devkit/helper/timeseries.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from t4_devkit import T4Devkit - from t4_devkit.schema import ObjectAnn, Sample, SampleAnnotation, SampleData + from t4_devkit.schema.v1 import ObjectAnn, Sample, SampleAnnotation, SampleData __all__ = ["TimeseriesHelper"] @@ -26,7 +26,7 @@ def __init__(self, t4: T4Devkit) -> None: self._sample_and_instance_to_ann3d: dict[tuple[str, str], str] = { (ann.sample_token, ann.instance_token): ann.token for ann in self._t4.sample_annotation } - self._sample_data_and_instance_to_ann2d: dict[tuple[str, str], str] = { + self._sample_data_and_instance_to_ann2d: dict[tuple[str, str | None], str] = { (ann.sample_data_token, ann.instance_token): ann.token for ann in self._t4.object_ann } @@ -54,7 +54,7 @@ def get_sample_annotations_until( anns: list[SampleAnnotation] = [] is_successor = seconds >= 0 current_sample_token = start_sample.next if is_successor else start_sample.prev - while current_sample_token != "": + while current_sample_token is not None: current_sample: Sample = self._t4.get("sample", current_sample_token) if abs(microseconds2seconds(current_sample.timestamp - start_sample.timestamp)) > abs( @@ -99,7 +99,7 @@ def get_object_anns_until( current_sample_data_token = ( start_sample_data.next if is_successor else start_sample_data.prev ) - while current_sample_data_token != "": + while current_sample_data_token is not None: current_sample_data: SampleData = self._t4.get("sample_data", current_sample_data_token) if abs( diff --git a/t4_devkit/schema/compatibility.py b/t4_devkit/schema/compatibility.py deleted file mode 100644 index 86e0b8a2..00000000 --- a/t4_devkit/schema/compatibility.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Compatibility helpers that abstract away differences between T4Dataset revisions.""" - -from __future__ import annotations - -from t4_devkit.schema import Category - - -def fix_category_table(categories: list[Category]) -> list[Category]: - """Fix unpopulated index fields in some T4Dataset revisions. - - This function behaves differently in the below three cases: - - - All `index` fields are set: the list is returned unmodified. - - All `index` fields are `None`: the position of each category in the list is used to compute - `index`. The resulting indices start at `0`. - - Some `index` fields are set and some are `None`: raise a `ValueError`. - - Args: - categories (list[Category]): List of categories to fix. - - Returns: - list[Category]: Fixed list of categories. - - Raises: - ValueError: If the `index` field is set for some categories and `None` for others. - """ - if any(category.index is None for category in categories): - if not all(category.index is None for category in categories): - raise ValueError("Category index is not set for some categories.") - - for idx, category in enumerate(categories): - category.index = idx - return categories diff --git a/t4_devkit/schema/v1/loader.py b/t4_devkit/schema/v1/loader.py index 39a92bfe..647de3b9 100644 --- a/t4_devkit/schema/v1/loader.py +++ b/t4_devkit/schema/v1/loader.py @@ -52,7 +52,26 @@ def get(self, name: str | SchemaName, token: str) -> SchemaTable: The record. """ name = SchemaName(name) - return self.tables[name][self._token2idx[name][token]] + return self.tables[name][self.get_idx(name, token)] + + def get_idx(self, name: str | SchemaName, token: str) -> int: + """Return the position of a record in its table by its token. + + Args: + name (str | SchemaName): Table name. + token (str): Record token. + + Returns: + The position in `tables[name]`. + + Raises: + KeyError: If no record of the table has this token. + """ + name = SchemaName(name) + idx = self._token2idx[name].get(token) + if idx is None: + raise KeyError(f"{token} is not registered in {name.value}.") + return idx def _fill_shortcuts(self) -> None: """Fill the shortcut fields (`init=False`) of the records, from their links.""" diff --git a/t4_devkit/tier4.py b/t4_devkit/tier4.py index e9cfa290..bb1ea950 100644 --- a/t4_devkit/tier4.py +++ b/t4_devkit/tier4.py @@ -15,37 +15,38 @@ from t4_devkit.common.geometry import is_box_in_image from t4_devkit.dataclass import Box2D, Box3D, SemanticLabel, Shape, ShapeType from t4_devkit.helper import RenderingHelper, TimeseriesHelper -from t4_devkit.schema import SchemaName, SensorModality, VisibilityLevel, build_schema -from t4_devkit.schema.compatibility import fix_category_table +from t4_devkit.schema import SensorModality, VisibilityLevel +from t4_devkit.schema.v1 import SchemaName, load_dataset if TYPE_CHECKING: + from t4_devkit.schema import SchemaTable from t4_devkit.typing import CameraIntrinsicLike, PathLike, Vector3 from .dataclass import BoxLike - from .schema import ( + from .schema.v1 import ( Attribute, - CalibratedSensor, + Caption, Category, EgoPose, Instance, Keypoint, - LidarSeg, Log, Map, ObjectAnn, + Route, Sample, SampleAnnotation, SampleData, + Scenario, Scene, - SchemaTable, Sensor, - SurfaceAnn, TrafficLight, + Vehicle, VehicleState, Visibility, ) -__all__ = ["DBMetadata", "load_metadata", "load_table", "Tier4", "T4Devkit"] +__all__ = ["DBMetadata", "load_metadata", "Tier4", "T4Devkit"] @define @@ -96,29 +97,6 @@ def load_metadata(db_root: PathLike, revision: str | None = None) -> DBMetadata: return DBMetadata(data_root=data_root, dataset_id=dataset_id, version=version) -def load_table(annotation_dir: str, schema: SchemaName) -> list[SchemaTable]: - """Load schema table from a JSON file. - - If the schema is optional and there is no corresponding JSON file in dataset, - returns empty list. - - Args: - annotation_dir (str): Path to the directory of JSON annotation schema files. - schema (SchemaName): An enum member of `SchemaName`. - - Returns: - Loaded table data saved in `.json`. - """ - filepath = osp.join(annotation_dir, schema.filename) - if not osp.exists(filepath) and schema.is_optional(): - return [] - - if not osp.exists(filepath): - raise FileNotFoundError(f"{schema.value} is mandatory.") - - return build_schema(schema, filepath) - - class T4Devkit: """Database class for T4 dataset to help query and retrieve information from the database.""" @@ -140,26 +118,16 @@ def __init__( >>> from t4_devkit import T4Devkit >>> t4 = T4Devkit("data/tier4") ====== - Loading T4 tables in `annotation`... - Reverse indexing... - Done reverse indexing in 0.010 seconds. - ====== - 21 category - 8 attribute - 4 visibility - 31 instance - 7 sensor - 7 calibrated_sensor - 2529 ego_pose - 1 log - 1 scene + Loading T4 tables... 88 sample 2529 sample_data - 1919 sample_annotation - 0 object_ann - 0 surface_ann - 0 keypoint + 2529 ego_pose + 7 sensor + 1 scene + 1 log + 1 vehicle 1 map + ... Done loading in 0.046 seconds. ====== @@ -169,48 +137,41 @@ def __init__( if not osp.exists(self.data_root): raise FileNotFoundError(f"Database directory is not found: {self.data_root}") - if self.version is None: + if self._metadata.version is None: warnings.warn( - f"DatasetID: {self.dataset_id} does't contain any versions.", DeprecationWarning + f"DatasetID: {self._metadata.dataset_id} does't contain any versions.", + DeprecationWarning, ) start_time = time.time() if verbose: print("======\nLoading T4 tables...") - # assign tables explicitly - self.attribute: list[Attribute] = load_table(self.annotation_dir, SchemaName.ATTRIBUTE) - self.calibrated_sensor: list[CalibratedSensor] = load_table( - self.annotation_dir, SchemaName.CALIBRATED_SENSOR - ) - self.category: list[Category] = load_table(self.annotation_dir, SchemaName.CATEGORY) - self.category = fix_category_table(self.category) - - self.ego_pose: list[EgoPose] = load_table(self.annotation_dir, SchemaName.EGO_POSE) - self.instance: list[Instance] = load_table(self.annotation_dir, SchemaName.INSTANCE) - self.keypoint: list[Keypoint] = load_table(self.annotation_dir, SchemaName.KEYPOINT) - self.lidarseg: list[LidarSeg] = load_table(self.annotation_dir, SchemaName.LIDARSEG) - self.log: list[Log] = load_table(self.annotation_dir, SchemaName.LOG) - self.map: list[Map] = load_table(self.annotation_dir, SchemaName.MAP) - self.object_ann: list[ObjectAnn] = load_table(self.annotation_dir, SchemaName.OBJECT_ANN) - self.sample_annotation: list[SampleAnnotation] = load_table( - self.annotation_dir, SchemaName.SAMPLE_ANNOTATION - ) - self.sample_data: list[SampleData] = load_table(self.annotation_dir, SchemaName.SAMPLE_DATA) - self.sample: list[Sample] = load_table(self.annotation_dir, SchemaName.SAMPLE) - self.scene: list[Scene] = load_table(self.annotation_dir, SchemaName.SCENE) - self.sensor: list[Sensor] = load_table(self.annotation_dir, SchemaName.SENSOR) - self.surface_ann: list[SurfaceAnn] = load_table(self.annotation_dir, SchemaName.SURFACE_ANN) - self.vehicle_state: list[VehicleState] = load_table( - self.annotation_dir, SchemaName.VEHICLE_STATE - ) - self.visibility: list[Visibility] = load_table(self.annotation_dir, SchemaName.VISIBILITY) - self.traffic_light: list[TrafficLight] = load_table( - self.annotation_dir, SchemaName.TRAFFIC_LIGHT - ) + # 0.x datasets are converted to 1.0 records on loading. + self._tables = load_dataset(self.data_root) + tables = self._tables.tables - # make reverse indexes for common lookups - self.__make_reverse_index__(verbose) + # assign tables explicitly + self.sample: list[Sample] = tables[SchemaName.SAMPLE] + self.sample_data: list[SampleData] = tables[SchemaName.SAMPLE_DATA] + self.ego_pose: list[EgoPose] = tables[SchemaName.EGO_POSE] + self.sensor: list[Sensor] = tables[SchemaName.SENSOR] + self.scene: list[Scene] = tables[SchemaName.SCENE] + self.log: list[Log] = tables[SchemaName.LOG] + self.vehicle: list[Vehicle] = tables[SchemaName.VEHICLE] + self.map: list[Map] = tables[SchemaName.MAP] + self.vehicle_state: list[VehicleState] = tables[SchemaName.VEHICLE_STATE] + self.category: list[Category] = tables[SchemaName.CATEGORY] + self.attribute: list[Attribute] = tables[SchemaName.ATTRIBUTE] + self.visibility: list[Visibility] = tables[SchemaName.VISIBILITY] + self.instance: list[Instance] = tables[SchemaName.INSTANCE] + self.sample_annotation: list[SampleAnnotation] = tables[SchemaName.SAMPLE_ANNOTATION] + self.object_ann: list[ObjectAnn] = tables[SchemaName.OBJECT_ANN] + self.keypoint: list[Keypoint] = tables[SchemaName.KEYPOINT] + self.traffic_light: list[TrafficLight] = tables[SchemaName.TRAFFIC_LIGHT] + self.route: list[Route] = tables[SchemaName.ROUTE] + self.scenario: list[Scenario] = tables[SchemaName.SCENARIO] + self.caption: list[Caption] = tables[SchemaName.CAPTION] if verbose: for schema in SchemaName: @@ -229,13 +190,16 @@ def data_root(self) -> str: @property def dataset_id(self) -> str: - """Return the dataset ID.""" - return self._metadata.dataset_id + """Return the dataset ID: from `manifest.json`, or the folder name for 0.x.""" + manifest = self._tables.manifest + return manifest.dataset_id if manifest is not None else self._metadata.dataset_id @property def version(self) -> str | None: - """Return the dataset version, or None if it is failed to lookup.""" - return self._metadata.version + """Return the dataset version: from `manifest.json`, or the folder name for 0.x (None if + there is no version folder).""" + manifest = self._tables.manifest + return str(manifest.version) if manifest is not None else self._metadata.version @property def annotation_dir(self) -> str: @@ -252,91 +216,6 @@ def bag_dir(self) -> str: """Return the path to ROS bag directory.""" return osp.join(self.data_root, "input_bag") - def __make_reverse_index__(self, verbose: bool) -> None: - """De-normalize database to create reverse indices for common cases. - - Args: - verbose (bool): Whether to display outputs. - - Raises: - ValueError: Expecting `map` table has `log_tokens` key. - """ - start_time = time.time() - if verbose: - print("Reverse indexing...") - - self._token2idx: dict[str, dict[str, int]] = { - schema.value: { - table.token: idx for idx, table in enumerate(self.get_table(schema.value)) - } - for schema in SchemaName - } - - self._label2id: dict[str, int] = { - category.name: category.index for category in self.category - } - - # add shortcuts - for record in self.sample_annotation: - instance: Instance = self.get("instance", record.instance_token) - category: Category = self.get("category", instance.category_token) - record.category_name = category.name - - for record in self.object_ann: - category: Category = self.get("category", record.category_token) - record.category_name = category.name - - for record in self.surface_ann: - if record.category_token == "": # NOTE: Some database contains this case - warnings.warn(f"Category token is empty for surface ann: {record.token}") - continue - category: Category = self.get("category", record.category_token) - record.category_name = category.name - - registered_channels: list[str] = [] - for record in self.sample_data: - cs_record: CalibratedSensor = self.get( - "calibrated_sensor", record.calibrated_sensor_token - ) - sensor_record: Sensor = self.get("sensor", cs_record.sensor_token) - record.modality = sensor_record.modality - record.channel = sensor_record.channel - # set first sample data token to the corresponding sensor channel, - # as premise for sample data is ordered by time stamp order. - if sensor_record.channel not in registered_channels: - sensor_record.first_sd_token = record.token - registered_channels.append(sensor_record.channel) - - # set sample data - if record.is_key_frame: - sample_record: Sample = self.get("sample", record.sample_token) - sample_record.data[record.channel] = record.token - - for ann_record in self.sample_annotation: - sample_record: Sample = self.get("sample", ann_record.sample_token) - sample_record.ann_3ds.append(ann_record.token) - - for ann_record in self.object_ann: - sd_record: SampleData = self.get("sample_data", ann_record.sample_data_token) - sample_record: Sample = self.get("sample", sd_record.sample_token) - sample_record.ann_2ds.append(ann_record.token) - - for ann_record in self.surface_ann: - sd_record: SampleData = self.get("sample_data", ann_record.sample_data_token) - sample_record: Sample = self.get("sample", sd_record.sample_token) - sample_record.surface_anns.append(ann_record.token) - - log_to_map: dict[str, str] = {} - for map_record in self.map: - for log_token in map_record.log_tokens: - log_to_map[log_token] = map_record.token - for log_record in self.log: - log_record.map_token = log_to_map[log_record.token] - - if verbose: - elapsed_time = time.time() - start_time - print(f"Done reverse indexing in {elapsed_time:.3f} seconds.\n======") - def get_table(self, schema: str | SchemaName) -> list[SchemaTable]: """Return the list of dataclasses corresponding to the schema table. @@ -346,7 +225,7 @@ def get_table(self, schema: str | SchemaName) -> list[SchemaTable]: Returns: List of dataclasses. """ - return getattr(self, SchemaName(schema)) + return self._tables.tables[SchemaName(schema)] def get(self, schema: str | SchemaName, token: str) -> SchemaTable: """Return a record identified by the associated token. @@ -358,7 +237,7 @@ def get(self, schema: str | SchemaName, token: str) -> SchemaTable: Returns: Table record of the corresponding token. """ - return self.get_table(schema)[self.get_idx(schema, token)] + return self._tables.get(schema, token) def get_idx(self, schema: str | SchemaName, token: str) -> int: """Return the index of the record in a table in constant runtime. @@ -370,12 +249,7 @@ def get_idx(self, schema: str | SchemaName, token: str) -> int: Returns: The index of the record in table. """ - schema = SchemaName(schema) - if self._token2idx.get(schema) is None: - raise KeyError(f"{schema} is not registered.") - if self._token2idx[schema].get(token) is None: - raise KeyError(f"{token} is not registered in {schema}.") - return self._token2idx[schema][token] + return self._tables.get_idx(schema, token) def get_sample_data_path(self, sample_data_token: str) -> str: """Return the file path to a raw data recorded in `sample_data`. @@ -416,17 +290,14 @@ def get_sample_data( """ # Retrieve sensor & pose records sd_record: SampleData = self.get("sample_data", sample_data_token) - cs_record: CalibratedSensor = self.get( - "calibrated_sensor", sd_record.calibrated_sensor_token - ) - sensor_record: Sensor = self.get("sensor", cs_record.sensor_token) + sensor_record: Sensor = self.get("sensor", sd_record.sensor_token) pose_record: EgoPose = self.get("ego_pose", sd_record.ego_pose_token) data_path = self.get_sample_data_path(sample_data_token) if sensor_record.modality == SensorModality.CAMERA: - cam_intrinsic = cs_record.camera_intrinsic - img_size = (sd_record.width, sd_record.height) + cam_intrinsic = sensor_record.camera.intrinsic + img_size = (sensor_record.camera.width, sensor_record.camera.height) else: cam_intrinsic = None img_size = None @@ -462,8 +333,8 @@ def get_sample_data( if as_sensor_coord: # Move box to sensor coord system. - box.translate(-cs_record.translation) - box.rotate(cs_record.rotation.inverse) + box.translate(-sensor_record.translation) + box.rotate(sensor_record.rotation.inverse) box.frame_id = sensor_record.channel if sensor_record.modality == SensorModality.CAMERA and not is_box_in_image( @@ -513,7 +384,6 @@ def get_box3d(self, sample_annotation_token: str, *, future_seconds: float = 0.0 ann: SampleAnnotation = self.get("sample_annotation", sample_annotation_token) instance: Instance = self.get("instance", ann.instance_token) sample: Sample = self.get("sample", ann.sample_token) - visibility: Visibility = self.get("visibility", ann.visibility_token) # semantic label semantic_label = self.get_semantic_label( @@ -534,10 +404,10 @@ def get_box3d(self, sample_annotation_token: str, *, future_seconds: float = 0.0 rotation=ann.rotation, shape=shape, velocity=velocity, - confidence=1.0, - uuid=instance.token, # TODO(ktro2828): extract uuid from `instance_name`. + confidence=self._box_confidence(ann), + uuid=instance.token, # TODO(ktro2828): extract uuid from `instance.name`. num_points=ann.num_lidar_pts, - visibility=visibility.level, + visibility=self._box_visibility(ann), ) if future_seconds > 0.0: @@ -562,7 +432,6 @@ def get_box2d(self, object_ann_token: str) -> Box2D: Instantiated Box2D. """ ann: ObjectAnn = self.get("object_ann", object_ann_token) - instance: Instance = self.get("instance", ann.instance_token) sample_data: SampleData = self.get("sample_data", ann.sample_data_token) semantic_label = self.get_semantic_label( @@ -575,13 +444,16 @@ def get_box2d(self, object_ann_token: str) -> Box2D: frame_id=sample_data.channel, semantic_label=semantic_label, roi=ann.bbox, - confidence=1.0, - uuid=instance.token, # TODO(ktro2828): extract uuid from `instance_name`. + confidence=self._box_confidence(ann), + # regions without an instance (road, sky) have no uuid + uuid=ann.instance_token, # TODO(ktro2828): extract uuid from `instance.name`. ) def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> list[Box3D]: """Rerun a list of Box3D classes for all annotations of a particular `sample_data` record. - It the `sample_data` is a keyframe, this returns annotations for the corresponding `sample`. + It the `sample` of the `sample_data` has a key frame, this returns its annotations. + Otherwise, boxes are interpolated between the next key sample and the previous one; after + the last key sample, the boxes of the last one are returned. Args: sample_data_token (str): Token of `sample_data`. @@ -592,18 +464,27 @@ def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> """ # Retrieve sensor & pose records sd_record: SampleData = self.get("sample_data", sample_data_token) - curr_sample_record: Sample = self.get("sample", sd_record.sample_token) + sample_record: Sample = self.get("sample", sd_record.sample_token) + curr_sample_record = self._key_sample(sample_record.token, forward=True) + after_last_key_sample = curr_sample_record is None + if after_last_key_sample: + curr_sample_record = self._key_sample(sample_record.token, forward=False) + if curr_sample_record is None: + return [] + prev_sample_record = ( + None + if sample_record.has_key_frame or after_last_key_sample + else self._key_sample(curr_sample_record.prev, forward=False) + ) - if curr_sample_record.prev == "" or sd_record.is_key_frame: - # If no previous annotations available, or if sample_data is keyframe just return the current ones. + if prev_sample_record is None: + # A key sample, or no key sample on one side: just return the key sample's boxes. boxes = [ self.get_box3d(token, future_seconds=future_seconds) for token in curr_sample_record.ann_3ds ] else: - prev_sample_record: Sample = self.get("sample", curr_sample_record.prev) - curr_ann_recs: list[SampleAnnotation] = [ self.get("sample_annotation", token) for token in curr_sample_record.ann_3ds ] @@ -649,7 +530,6 @@ def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> instance.category_token, curr_ann.attribute_tokens ) velocity = self.box_velocity(curr_ann.token) - visibility: Visibility = self.get("visibility", curr_ann.visibility_token) box = Box3D( unix_time=t, @@ -659,10 +539,10 @@ def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> rotation=rotation, shape=Shape(ShapeType.BOUNDING_BOX, curr_ann.size), velocity=velocity, - confidence=1.0, - uuid=instance.token, # TODO(ktro2828): extract uuid from `instance_name`. + confidence=self._box_confidence(curr_ann), + uuid=instance.token, # TODO(ktro2828): extract uuid from `instance.name`. num_points=curr_ann.num_lidar_pts, - visibility=visibility.level, + visibility=self._box_visibility(curr_ann), ) else: # If not, simply grab the current annotation. @@ -710,8 +590,8 @@ def box_velocity(self, sample_annotation_token: str, max_time_diff: float = 1.5) if current.velocity is not None: return current.velocity - has_prev = current.prev != "" - has_next = current.next != "" + has_prev = current.prev is not None + has_next = current.next is not None # Cannot estimate velocity for a single annotation. if not has_prev and not has_next: @@ -814,6 +694,28 @@ def render_pointcloud( show_map=show_map, ) + def _key_sample(self, sample_token: str | None, *, forward: bool) -> Sample | None: + """The first sample with a key frame from a sample (included), following `next` or + `prev`, or None.""" + while sample_token is not None: + sample: Sample = self.get("sample", sample_token) + if sample.has_key_frame: + return sample + sample_token = sample.next if forward else sample.prev + return None + + def _box_visibility(self, ann: SampleAnnotation) -> VisibilityLevel: + """Visibility of a 3D box; `UNAVAILABLE` when the annotation's visibility is unknown.""" + if ann.visibility_token is None: + return VisibilityLevel.UNAVAILABLE + visibility: Visibility = self.get("visibility", ann.visibility_token) + return VisibilityLevel(visibility.level.value) + + @staticmethod + def _box_confidence(ann: SampleAnnotation | ObjectAnn) -> float: + """Confidence of a box: the label's, or 1.0 for labels without one (manual).""" + return 1.0 if ann.confidence is None else ann.confidence + @deprecated("Tier4 is deprecated. Use T4Devkit instead.") class Tier4(T4Devkit): diff --git a/t4_devkit/viewer/viewer.py b/t4_devkit/viewer/viewer.py index 72a73b5e..3fdd41ef 100644 --- a/t4_devkit/viewer/viewer.py +++ b/t4_devkit/viewer/viewer.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from t4_devkit.dataclass import Box2D, Box3D, Future, PointCloudLike - from t4_devkit.schema import CalibratedSensor, EgoPose, Sensor + from t4_devkit.schema.v1 import EgoPose, Sensor from t4_devkit.typing import ( CameraIntrinsicLike, NDArrayU8, @@ -545,18 +545,12 @@ def _render_ego_without_schema( rr.log(entity_path, rr.GeoPoints(lat_lon=(latitude, longitude))) @overload - def render_calibration( - self, - sensor: Sensor, - calibration: CalibratedSensor, - resolution: Vector2Like | None = None, - ) -> None: + def render_calibration(self, sensor: Sensor) -> None: """Render a sensor calibration. Args: - sensor (Sensor): `Sensor` object. - calibration (CalibratedSensor): `CalibratedSensor` object. - resolution (Vector2Like | None, optional): Camera resolution (width, height). + sensor (Sensor): `Sensor` record, with its extrinsics and, for a camera, its + intrinsics and image size. """ pass @@ -585,24 +579,20 @@ def render_calibration( @_check_spatial3d def render_calibration(self, *args, **kwargs) -> None: """Render a sensor calibration.""" - if len(args) + len(kwargs) <= 3: + if len(args) + len(kwargs) == 1: self._render_calibration_with_schema(*args, **kwargs) else: self._render_calibration_without_schema(*args, **kwargs) - def _render_calibration_with_schema( - self, - sensor: Sensor, - calibration: CalibratedSensor, - resolution: Vector2Like | None = None, - ) -> None: + def _render_calibration_with_schema(self, sensor: Sensor) -> None: + camera = sensor.camera self._render_calibration_without_schema( channel=sensor.channel, modality=sensor.modality, - translation=calibration.translation, - rotation=calibration.rotation, - camera_intrinsic=calibration.camera_intrinsic, - resolution=resolution, + translation=sensor.translation, + rotation=sensor.rotation, + camera_intrinsic=None if camera is None else camera.intrinsic, + resolution=None if camera is None else (camera.width, camera.height), ) def _render_calibration_without_schema( diff --git a/tests/sample/t4dataset_v1/annotation/object_ann.json b/tests/sample/t4dataset_v1/annotation/object_ann.json index b875c5a7..000a2bc7 100644 --- a/tests/sample/t4dataset_v1/annotation/object_ann.json +++ b/tests/sample/t4dataset_v1/annotation/object_ann.json @@ -22,7 +22,7 @@ "bbox": [0.0, 24.0, 64.0, 48.0], "mask": { "size": [48, 64], - "counts": "0" + "counts": "aDBoMGgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=" }, "orientation": null, "number": null, diff --git a/tests/sample/t4dataset_v1/make_sample_v1.py b/tests/sample/t4dataset_v1/make_sample_v1.py index 570235cf..ec7ccc36 100644 --- a/tests/sample/t4dataset_v1/make_sample_v1.py +++ b/tests/sample/t4dataset_v1/make_sample_v1.py @@ -12,6 +12,7 @@ from __future__ import annotations +import base64 import hashlib import json import os @@ -21,6 +22,7 @@ import numpy as np import pyarrow as pa import pyarrow.parquet as pq +from pycocotools import mask as cocomask ROOT = Path(__file__).parent FFMPEG = os.environ.get("FFMPEG", "ffmpeg") @@ -49,6 +51,14 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def road_mask() -> dict[str, object]: + """RLE mask of the road: the bottom half of the image (rows 24-47), as pycocotools writes it.""" + mask = np.zeros((HEIGHT, WIDTH), dtype=np.uint8, order="F") + mask[24:, :] = 1 + rle = cocomask.encode(mask) + return {"size": [HEIGHT, WIDTH], "counts": base64.b64encode(rle["counts"]).decode()} + + def write_camera() -> None: """Encode NUM_FRAMES synthetic RGB frames as intra-only AV1 with pts == frame_index.""" frames = np.zeros((NUM_FRAMES, HEIGHT, WIDTH, 3), dtype=np.uint8) @@ -395,7 +405,7 @@ def link(tokens: list[str], i: int) -> dict: "category_token": "category_road", "attribute_tokens": [], "bbox": [0.0, 24.0, float(WIDTH), float(HEIGHT)], - "mask": {"size": [HEIGHT, WIDTH], "counts": "0"}, + "mask": road_mask(), "orientation": None, "number": None, **label(), diff --git a/tests/sample/t4dataset_v1/manifest.json b/tests/sample/t4dataset_v1/manifest.json index 92d5d003..669c53d7 100644 --- a/tests/sample/t4dataset_v1/manifest.json +++ b/tests/sample/t4dataset_v1/manifest.json @@ -88,7 +88,7 @@ }, "object_ann": { "path": "annotation/object_ann.json", - "sha256": "37d5ad8200e17bcc0e6fdc4f7c47648f33a6a21b5fa115830751f1b4eec39dba" + "sha256": "d3a84be002e1b8db8ed98d4e547e8baf6a43b1aa101ba6fcd3773a70ddfc7c9e" }, "keypoint": { "path": "annotation/keypoint.json", diff --git a/tests/test_tier4.py b/tests/test_tier4.py index 145a60ab..a8d5460c 100644 --- a/tests/test_tier4.py +++ b/tests/test_tier4.py @@ -1,6 +1,8 @@ from __future__ import annotations +import json import os +import shutil from pathlib import Path from unittest.mock import patch @@ -8,8 +10,8 @@ import pytest from t4_devkit.dataclass import Box2D, Box3D, SemanticLabel -from t4_devkit.schema import SchemaName -from t4_devkit.tier4 import DBMetadata, T4Devkit, load_metadata, load_table +from t4_devkit.schema import VisibilityLevel +from t4_devkit.tier4 import DBMetadata, T4Devkit, load_metadata from t4_devkit.typing import Quaternion @@ -67,30 +69,46 @@ def test_load_metadata_with_specific_revision(self, tmp_path): assert metadata.version == "v2.0.0" -class TestLoadTable: - """Test cases for load_table function.""" - - def test_load_table_success(self, sample_dataset_path): - """Test successful table loading.""" - if not sample_dataset_path.exists(): - pytest.skip("Sample dataset not available") - - # Create a dummy annotation file - annotation_dir = sample_dataset_path.joinpath("annotation") - - result = load_table(annotation_dir.as_posix(), SchemaName.ATTRIBUTE) - - # The result should be the return value of the schema class, not its length - assert isinstance(result, list) - - def test_load_table_file_not_exists(self, tmp_path): - """Test table loading when file doesn't exist for optional schema.""" - annotation_dir = tmp_path / "annotation" - annotation_dir.mkdir() - - # Use an optional schema for this test (keypoint is actually optional) - result = load_table(str(annotation_dir), SchemaName.KEYPOINT) - assert result == [] +def test_unknown_visibility(tmp_path): + """A box whose visibility is unknown (0.x `unavailable`) gets `UNAVAILABLE`.""" + data_root = tmp_path / "t4dataset" + shutil.copytree(Path(__file__).parent / "sample/t4dataset", data_root) + visibility_path = data_root / "annotation" / "visibility.json" + visibilities = json.loads(visibility_path.read_text()) + visibilities[0]["level"] = "unavailable" # the level of the first annotation + visibility_path.write_text(json.dumps(visibilities)) + + t4 = T4Devkit(data_root.as_posix(), verbose=False) + annotation = t4.sample_annotation[0] + assert annotation.visibility_token is None + assert t4.get_box3d(annotation.token).visibility == VisibilityLevel.UNAVAILABLE + + +def test_autolabel_confidence(tmp_path): + """A box takes the confidence of its label; manual labels keep 1.0.""" + data_root = tmp_path / "t4dataset" + shutil.copytree(Path(__file__).parent / "sample/t4dataset", data_root) + annotation_path = data_root / "annotation" / "sample_annotation.json" + annotations = json.loads(annotation_path.read_text()) + annotations[0]["automatic_annotation"] = True + annotations[0]["autolabel_metadata"] = [{"name": "centerpoint-v2", "score": 0.8}] + annotation_path.write_text(json.dumps(annotations)) + + t4 = T4Devkit(data_root.as_posix(), verbose=False) + assert t4.get_box3d(t4.sample_annotation[0].token).confidence == 0.8 + assert t4.get_box3d(t4.sample_annotation[1].token).confidence == 1.0 + + +def test_non_key_sample_boxes(): + """A frame after the last key sample gets that sample's boxes; a key frame its own.""" + t4 = T4Devkit((Path(__file__).parent / "sample/t4dataset_v1").as_posix(), verbose=False) + non_key = next( + sd for sd in t4.sample_data if not t4.get("sample", sd.sample_token).has_key_frame + ) + last_key = t4.get("sample", t4.get("sample", non_key.sample_token).prev) + assert last_key.has_key_frame + boxes = t4.get_box3ds(non_key.token) + assert len(boxes) == len(last_key.ann_3ds) > 0 class TestT4Devkit: @@ -106,25 +124,48 @@ class TestT4Devkit: - Testing performance with realistic data sizes """ - @pytest.fixture(scope="class") - def sample_t4(self): - """Create a T4Devkit instance using the real sample dataset.""" - sample_dataset_path = Path(__file__).parent / "sample/t4dataset" - if not sample_dataset_path.exists(): - pytest.skip("Sample dataset not available") - return T4Devkit(sample_dataset_path.as_posix(), verbose=False) + # Expected contents of each sample dataset: the 0.x one (converted on loading) and the 1.0 one. + EXPECTED = { + "t4dataset": { + "samples": 3, + "sample_data": 9, + "categories": 4, + "cameras": 2, + "scene_name": "minimal_test_scene", + "dataset_id": "t4dataset", + "version": None, + }, + "t4dataset_v1": { + "samples": 3, + "sample_data": 5, + "categories": 2, + "cameras": 1, + "scene_name": "sample_scene", + "dataset_id": "sample", + "version": "0", + }, + } + + @pytest.fixture(scope="class", params=sorted(EXPECTED)) + def sample_t4(self, request): + """Create a T4Devkit instance for each sample dataset.""" + sample_dataset_path = Path(__file__).parent / "sample" / request.param + t4 = T4Devkit(sample_dataset_path.as_posix(), verbose=False) + t4.expected = self.EXPECTED[request.param] + return t4 def test_dataset_structure_validation(self, sample_t4): """Validate the complete structure of the sample dataset.""" t4 = sample_t4 + expected = t4.expected - # Verify expected data counts from status.json assert len(t4.scene) == 1, "Expected exactly 1 scene" - assert len(t4.sample) == 3, "Expected exactly 3 samples" - assert len(t4.sample_data) == 9, "Expected exactly 9 sample data entries" - assert len(t4.category) == 4, "Expected exactly 4 categories" - assert len(t4.sensor) == 3, "Expected exactly 3 sensors" - assert len(t4.instance) >= 3, "Expected at least 3 instances" + assert len(t4.sample) == expected["samples"] + assert len(t4.sample_data) == expected["sample_data"] + assert len(t4.category) == expected["categories"] + assert len(t4.instance) >= 1, "Expected at least 1 instance" + assert t4.dataset_id == expected["dataset_id"] + assert t4.version == expected["version"] # Verify sensor modalities sensor_modalities = {sensor.modality for sensor in t4.sensor} @@ -136,7 +177,7 @@ def test_dataset_structure_validation(self, sample_t4): # Verify camera count camera_count = sum(1 for sensor in t4.sensor if sensor.modality == "camera") lidar_count = sum(1 for sensor in t4.sensor if sensor.modality == "lidar") - assert camera_count == 2, "Expected 2 cameras" + assert camera_count == expected["cameras"] assert lidar_count == 1, "Expected 1 LiDAR" def test_schema_reference_integrity(self, sample_t4): @@ -151,8 +192,8 @@ def test_schema_reference_integrity(self, sample_t4): for sample_data in t4.sample_data: _ = t4.get("sample", sample_data.sample_token) - # Test calibrated_sensor reference - _ = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) + # Test sensor reference + _ = t4.get("sensor", sample_data.sensor_token) # Test ego_pose reference _ = t4.get("ego_pose", sample_data.ego_pose_token) @@ -166,7 +207,8 @@ def test_schema_reference_integrity(self, sample_t4): _ = t4.get("instance", annotation.instance_token) # Test visibility reference - _ = t4.get("visibility", annotation.visibility_token) + if annotation.visibility_token is not None: + _ = t4.get("visibility", annotation.visibility_token) # Test instance -> category references for instance in t4.instance: @@ -185,8 +227,7 @@ def test_complete_sample_processing_workflow(self, sample_t4): # Process each sensor type for sample_data in sample_data_list: # Get sensor information - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) # Test sample data retrieval path, boxes, intrinsic = t4.get_sample_data(sample_data.token) @@ -196,8 +237,8 @@ def test_complete_sample_processing_workflow(self, sample_t4): # Verify sensor-specific properties if sensor.modality == "camera": assert intrinsic is not None, "Camera should have intrinsic matrix" - assert sample_data.width > 0, "Camera should have width" - assert sample_data.height > 0, "Camera should have height" + assert sensor.camera.width > 0, "Camera should have width" + assert sensor.camera.height > 0, "Camera should have height" elif sensor.modality == "lidar": assert intrinsic is None, "LiDAR should not have intrinsic matrix" @@ -243,6 +284,14 @@ def test_annotation_processing_workflows(self, sample_t4): assert hasattr(box2d, "semantic_label"), "Box should have semantic label" assert isinstance(box2d.semantic_label, SemanticLabel), "Label should be SemanticLabel" + def test_future_trajectory(self, sample_t4): + """A box gets a future path when a later annotation of its instance exists.""" + t4 = sample_t4 + + for annotation in t4.sample_annotation: + box = t4.get_box3d(annotation.token, future_seconds=10.0) + assert (box.future is not None) == (annotation.next is not None) + def test_keyframe_data_retrieval(self, sample_t4): """Test keyframe-specific data retrieval.""" t4 = sample_t4 @@ -255,8 +304,7 @@ def test_keyframe_data_retrieval(self, sample_t4): # Test 3D boxes for keyframe using sample data for sample_data in keyframe_sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) if sensor.modality == "lidar": boxes_3d = t4.get_box3ds(sample_data.token) @@ -268,8 +316,7 @@ def test_keyframe_data_retrieval(self, sample_t4): # Test 2D boxes for camera data for sample_data in keyframe_sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) if sensor.modality == "camera": boxes_2d = t4.get_box2ds(sample_data.token) @@ -312,52 +359,29 @@ def test_coordinate_system_consistency(self, sample_t4): t4 = sample_t4 for sample_data in t4.sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) ego_pose = t4.get("ego_pose", sample_data.ego_pose_token) - # Verify coordinate transformations exist - assert hasattr( - calibrated_sensor, "translation" - ), "Calibrated sensor should have translation" - assert hasattr(calibrated_sensor, "rotation"), "Calibrated sensor should have rotation" - assert hasattr(ego_pose, "translation"), "Ego pose should have translation" - assert hasattr(ego_pose, "rotation"), "Ego pose should have rotation" - # Verify coordinate dimensions - assert len(calibrated_sensor.translation) == 3, "Translation should be 3D" - assert isinstance( - calibrated_sensor.rotation, Quaternion - ), "Rotation should be quaternion" + assert len(sensor.translation) == 3, "Translation should be 3D" + assert isinstance(sensor.rotation, Quaternion), "Rotation should be quaternion" assert len(ego_pose.translation) == 3, "Translation should be 3D" assert isinstance(ego_pose.rotation, Quaternion), "Rotation should be quaternion" - def test_rendering_methods_integration(self, sample_t4): - """Test that rendering methods work with real data (without actually rendering).""" + def test_rendering(self, sample_t4, tmp_path): + """Scene, instance and pointcloud render to recording files (no viewer window).""" t4 = sample_t4 + save_dirs = {name: tmp_path / name for name in ("scene", "instance", "pointcloud")} + for save_dir in save_dirs.values(): + save_dir.mkdir() - # Test that methods exist and are callable - assert hasattr(t4, "render_scene") and callable(t4.render_scene) - assert hasattr(t4, "render_instance") and callable(t4.render_instance) - assert hasattr(t4, "render_pointcloud") and callable(t4.render_pointcloud) + t4.render_scene(future_seconds=1.0, save_dir=str(save_dirs["scene"])) + t4.render_instance(t4.instance[0].token, save_dir=str(save_dirs["instance"])) + t4.render_pointcloud(save_dir=str(save_dirs["pointcloud"])) - # We don't actually call these methods to avoid opening GUI windows - # but we can test that the data they would need is available - - # For render_scene: need sample data - assert len(t4.sample_data) > 0, "Need sample data for scene rendering" - - # For render_instance: need instances and annotations - assert len(t4.instance) > 0, "Need instances for instance rendering" - assert len(t4.sample_annotation) > 0, "Need annotations for instance rendering" - - # For render_pointcloud: need LiDAR data - lidar_data = [] - for sd in t4.sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sd.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) - if sensor.modality == "lidar": - lidar_data.append(sd) - assert len(lidar_data) > 0, "Need LiDAR data for pointcloud rendering" + for save_dir in save_dirs.values(): + recordings = list(save_dir.glob("*.rrd")) + assert recordings and all(path.stat().st_size > 0 for path in recordings) def test_semantic_label_consistency(self, sample_t4): """Test semantic label consistency across annotations.""" @@ -419,7 +443,7 @@ def test_data_file_paths(self, sample_t4): assert path.startswith(t4.data_root), f"Path should start with data root: {path}" # Should end with expected file extension - expected_extensions = [".jpg", ".png", ".pcd.bin", ".bin"] + expected_extensions = [".jpg", ".png", ".pcd.bin", ".bin", ".mp4", ".parquet"] assert any( path.endswith(ext) for ext in expected_extensions ), f"Unexpected file extension: {path}" @@ -434,7 +458,7 @@ def test_complete_integration_workflow(self, sample_t4): # Start with scene scene = t4.scene[0] - assert scene.name == "minimal_test_scene" + assert scene.name == t4.expected["scene_name"] # Get first sample first_sample = t4.get("sample", scene.first_sample_token) @@ -447,8 +471,7 @@ def test_complete_integration_workflow(self, sample_t4): # Process camera data camera_data = None for sd in sample_data_list: - calibrated_sensor = t4.get("calibrated_sensor", sd.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sd.sensor_token) if sensor.modality == "camera": camera_data = sd break @@ -456,7 +479,7 @@ def test_complete_integration_workflow(self, sample_t4): if camera_data: # Get camera sample data path, boxes, intrinsic = t4.get_sample_data(camera_data.token) - assert path.endswith(".jpg") + assert path.endswith((".jpg", ".mp4")) assert intrinsic is not None # Get 2D boxes for this camera @@ -466,8 +489,7 @@ def test_complete_integration_workflow(self, sample_t4): # Process LiDAR data lidar_data = None for sd in sample_data_list: - calibrated_sensor = t4.get("calibrated_sensor", sd.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sd.sensor_token) if sensor.modality == "lidar": lidar_data = sd break @@ -475,7 +497,7 @@ def test_complete_integration_workflow(self, sample_t4): if lidar_data: # Get LiDAR sample data path, boxes, intrinsic = t4.get_sample_data(lidar_data.token) - assert path.endswith((".pcd.bin", ".bin")) + assert path.endswith((".pcd.bin", ".bin", ".parquet")) assert intrinsic is None # Get 3D boxes for this sample using LiDAR data diff --git a/tests/viewer/test_viewer.py b/tests/viewer/test_viewer.py index d69e25e2..d3a8bc44 100644 --- a/tests/viewer/test_viewer.py +++ b/tests/viewer/test_viewer.py @@ -1,14 +1,21 @@ from __future__ import annotations +from pathlib import Path + import numpy as np import pytest from pyquaternion import Quaternion from t4_devkit.dataclass import LidarPointCloud -from t4_devkit.schema import CalibratedSensor, EgoPose, Sensor +from t4_devkit.schema import EgoPose +from t4_devkit.schema.v1 import Sensor from t4_devkit.viewer import EntityPath, format_entity from t4_devkit.viewer import viewer as viewer_module +SAMPLE_V1_SENSOR = ( + Path(__file__).parents[1] / "sample" / "t4dataset_v1" / "annotation" / "sensor.json" +) + def test_format_entity() -> None: """Test `format_entity(...)` function.""" @@ -153,17 +160,10 @@ def test_render_calibration(dummy_viewer, dummy_camera_calibration) -> None: camera_intrinsic=camera_intrinsic, ) - # with `Sensor` and `CalibratedSensor` - sensor = Sensor(token="sensor", channel="camera", modality="camera") - calibration = CalibratedSensor( - token="sensor_calibration", - sensor_token="sensor", - translation=translation, - rotation=rotation, - camera_intrinsic=camera_intrinsic, - camera_distortion=[0, 0, 0, 0, 0], - ) - dummy_viewer.render_calibration(sensor, calibration) + # with a `Sensor` record (camera of the 1.0 sample) + sensor = Sensor.from_json(SAMPLE_V1_SENSOR)[0] + assert sensor.camera is not None + dummy_viewer.render_calibration(sensor) def test_render_map(dummy_viewer, dummy_lanelet_path) -> None: