diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d4d74d8..9a004ca5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,6 +32,8 @@ repos: rev: v4.0.0-alpha.8 hooks: - id: prettier + # written by DatasetWriter: manifest.json holds the hashes of these files + exclude: ^tests/sample/t4dataset_v1/ - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.6.8 diff --git a/docs/cli/index.md b/docs/cli/index.md index 7f7d48d6..709a3b2b 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -4,3 +4,4 @@ Following command line tools are supported: - [`t4viz`: Visualize T4 dataset features](./t4viz.md) - [`t4sanity`: Sanity checker of T4 dataset](./t4sanity.md) +- [`t4migrate`: Migrate a 0.x dataset to 1.0](./t4migrate.md) diff --git a/docs/cli/t4migrate.md b/docs/cli/t4migrate.md new file mode 100644 index 00000000..46d4e86f --- /dev/null +++ b/docs/cli/t4migrate.md @@ -0,0 +1,43 @@ +`t4migrate` migrates a 0.x dataset to a new [1.0](../schema/v1/index.md) dataset. The 0.x dataset is only read. + +```shell +$ t4migrate -h + + Usage: t4migrate [OPTIONS] {data_root} {output_dir} + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ * data_root 0.x dataset: the / directory with annotation/. [required] │ +│ * output_dir Parent directory: the dataset goes to //. │ +│ [required] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --version -v Show the application version and exit. │ +│ * --dataset-version -d Version of the 1.0 dataset (given by Web.Auto). [required] │ +│ --dataset-id Dataset ID, if not the 0.x dataset's. │ +│ --keep-images Copy camera images as they are instead of encoding one AV1 video per │ +│ channel. │ +│ --help -h Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +## Usage + +```shell +t4migrate <0.x DATASET_ID>/ --dataset-version +``` + +This writes `//`, prints the migration notes, and runs [`t4sanity`](./t4sanity.md) on the result. It exits with `1` if a sanity rule fails. + +The dataset version is given by the caller: Web.Auto assigns it when the new version is uploaded. An existing output folder is never overwritten. + +## What Changes + +- Tables are converted as described in [Loading 0.x Datasets](../schema/v1/index.md#loading-0x-datasets). +- Each LiDAR channel becomes one Parquet file, one row group per frame: + - fields get their 1.0 types (e.g. `intensity` uint8) when every value fits; otherwise they keep their 0.x type, with a warning and a migration note; + - a ring (`channel`) of `-1` on every point (not recorded) is dropped, with a note; + - `.info` files become the `source_index` / `source_timestamp` columns; + - lidarseg becomes the `label` / `instance` columns (`instance` is `-1`: 0.x labels are semantic only). Category `0` must be `unpainted`. +- Each camera channel becomes one AV1 video, frames in timestamp order (or the images are copied with `--keep-images`). Frames with `is_valid: false` are encoded as they are. +- `map/` and `input_bag/` are copied as they are. +- `manifest.json` has `migrated_from_schema_version: "0.x"` and the migration notes. Provenance that 0.x does not store (Web.Auto IDs, topics) is empty, for backfilling. diff --git a/docs/cli/t4sanity.md b/docs/cli/t4sanity.md index ee5209bf..cbecaf16 100644 --- a/docs/cli/t4sanity.md +++ b/docs/cli/t4sanity.md @@ -1,4 +1,4 @@ -`t4sanity` performs sanity checks on T4 datasets, reporting any issues regarding the [dataset requirements](../schema/requirement.md). +`t4sanity` performs sanity checks on T4 datasets, reporting any issues regarding the dataset requirements: [0.x](../schema/requirement.md) or [1.0](../schema/v1/requirement.md), chosen by the dataset (1.0 if it has a `manifest.json`). ```shell $ t4sanity -h @@ -15,6 +15,7 @@ $ t4sanity -h │ --exclude -e TEXT Exclude specific rules or rule groups. [default: None] │ │ --strict -s Indicates whether warnings are treated as failures. │ │ --fix -f Attempt to fix the issues reported by the sanity check. │ +│ --checksum -c Verify the SHA-256 of every file listed in manifest.json (1.0 datasets). │ │ --install-completion Install completion for the current shell. │ │ --show-completion Show completion for the current shell, to copy it or customize the installation. │ │ --help -h Show this message and exit. │ diff --git a/docs/cli/t4viz.md b/docs/cli/t4viz.md index fc953d8c..d046621d 100644 --- a/docs/cli/t4viz.md +++ b/docs/cli/t4viz.md @@ -58,7 +58,7 @@ t4viz instance ... [OPTIONS] This command performs the same behavior with [`T4Devkit.render_pointcloud(...)`](../tutorials/render.md#rendering-pointcloud). -If a pointcloud `sample_data` record has `info_filename`, `t4viz pointcloud` loads the metainfo JSON automatically and uses `num_pts_feats` to parse extended point layouts. Without that file, it falls back to the standard 5-feature layout. +The point layout comes from the file (Parquet, BIN, PCD) or from `sensor.lidar.fields` (`.bin`; for 0.x datasets it is taken from the `.info` files when loading). For options, run `t4viz pointcloud -h`. diff --git a/docs/schema/requirement.md b/docs/schema/requirement.md index 8e5f8507..6f5ba575 100644 --- a/docs/schema/requirement.md +++ b/docs/schema/requirement.md @@ -1,5 +1,7 @@ # Dataset Requirements +The rules `t4sanity` checks on 0.x datasets. For T4Dataset 1.0, see [Dataset Requirements 1.0](./v1/requirement.md). + ## Structure (`STR`) | ID | Name | Severity | Fixable | Description | diff --git a/docs/schema/v1/data.md b/docs/schema/v1/data.md new file mode 100644 index 00000000..deffb5f9 --- /dev/null +++ b/docs/schema/v1/data.md @@ -0,0 +1,79 @@ +# Sensor Data 1.0 + +Each sensor channel is stored as a single file, `data//.`. A frame is addressed by `sample_data.filename` (the file) and `sample_data.frame_index` (the frame inside it), starting at 0. + +How the file is written is described on the channel's `sensor` row (`camera.video` or `lidar.container`, see [Schema Tables](./table.md#sensor)). + +## Camera + +One intra-only AV1 video per channel: + +```shell +data/ +└── CAM_FRONT/ + └── CAM_FRONT.mp4 +``` + +| Setting | Value | +| --------------- | ------------------------------------------------------------------ | +| Container | MP4 | +| Codec | AV1, encoded with SVT-AV1 (preset 6, CRF 26) | +| Frame types | Intra-only: every frame is a keyframe (`-g 1`) | +| Pixel format | `yuv420p`, full range, BT.601 | +| Track timescale | Equal to the frame rate, so a frame's pts equals its `frame_index` | + +Because every frame is a keyframe, any frame can be decoded without decoding its neighbours. Readers seek by `frame_index`. + +Datasets migrated from 0.x without re-encoding keep one image file per frame (`.jpg` or `.png`). In that case `sensor.camera.video` is `null` and `sample_data.frame_index` is `null`. + +## LiDAR + +One Parquet file per channel, with one row group per frame: + +```shell +data/ +└── LIDAR_CONCAT/ + └── LIDAR_CONCAT.parquet +``` + +- Columns are the point fields listed in `sensor.lidar.fields`, in that order, e.g. `x, y, z, intensity, ring`. +- Each row is one point. Row group `frame_index` holds all points of that frame. +- Compression is lossless (zstd), so values are bit-exact. +- Frame of reference is given in `sensor.lidar.frame_id` (`base_link` by default). + +### Point-wise Labels + +When `manifest.json` lists `segmentation3d` in `capabilities.tasks`, the LiDAR file has two extra columns for panoptic labels, listed in `sensor.lidar.fields` like any other field: + +| Column | Type | Value | +| ---------- | ------- | --------------------------------------------------------------------------------------------- | +| `label` | `uint8` | [`category.index`](./table.md#category) of the point's category; `0` for unpainted points | +| `instance` | `int16` | [`instance.index`](./table.md#instance) of the point's object; `-1` for points with no object | + +Each column reserves one value: + +- `label` `0` is the `unpainted` category: points the annotator did not label. When point-wise labels are present, `category.json` has a category named `unpainted` with `index` `0`. +- `instance` `-1` means no object: points of categories without `is_thing` (road, vegetation) and unpainted points. `instance.index` runs from `0` to `32767`. + +Labeled frames have a value for every point. Frames without labels (for example, non-key frames) have `null` in both columns for every point, because a Parquet file has the same columns in every frame. Readers leave out a column that is `null` for the whole frame. + +This replaces the `lidarseg` table and the `lidarseg/` folder of 0.x. + +### Merged Sources + +When `sensor.lidar.source_channels` is not empty, the channel merges several LiDARs and the file has two extra columns, listed in `sensor.lidar.fields` like any other field: + +| Column | Type | Value | +| ------------------ | ------- | -------------------------------------------------------------------- | +| `source_index` | `uint8` | [`sensor.index`](./table.md#sensor) of the LiDAR the point came from | +| `source_timestamp` | `int64` | Timestamp of that LiDAR's scan, in microseconds | + +Each source LiDAR has its own `sensor` row (with its calibration) and no `sample_data`. Points are grouped by source, so both columns are runs of equal values and cost almost nothing to store. + +This replaces the per-frame `.info` metainfo files of 0.x. + +0.x datasets loaded without migration keep one `.pcd.bin` or `.pcd` file per frame, as described in [Sensor Data](../data.md). In that case `lidar.container.format` is `bin` or `pcd` and `sample_data.frame_index` is `null`. + +## Radar + +Radar is not defined in 1.0. diff --git a/docs/schema/v1/index.md b/docs/schema/v1/index.md new file mode 100644 index 00000000..2107ba32 --- /dev/null +++ b/docs/schema/v1/index.md @@ -0,0 +1,185 @@ +# Dataset Schema 1.0 + +T4Dataset 1.0 is the next major version of the dataset format. Datasets written before 1.0 are referred to as **0.x** and are described in [Dataset Schema](../index.md). + +Main changes from 0.x: + +- Every dataset has a `manifest.json` that declares the schema version, identity, contents and checksums. It replaces `status.json`. +- `sensor` and `calibrated_sensor` are merged into a single `sensor` table with a modality-specific block (`camera` or `lidar`). +- `sample` lists every frame of the recording, not only the annotated ones; `sample.has_key_frame` and `sample_data.is_key_frame` mark the annotated ones. Every `sample_data` belongs to a sample. +- Camera images are stored as one AV1 video per channel, and LiDAR point clouds as one Parquet file per channel. `sample_data.frame_index` addresses a frame inside that file. +- Every label row records how it was made: `annotation_type`, `provenance` and `confidence`. +- New tables for planning and language data: `vehicle`, `route`, `scenario`, `caption`. `traffic_light` is keyed by the lanelet2 regulatory element. +- Missing references are `null` instead of an empty string `""`. + +## Version Detection + +A dataset is 1.0 or later if `manifest.json` exists at the dataset root. Otherwise it is a 0.x dataset. + +Readers must read `manifest.json` first and check `schema_version` before reading any table. A reader that does not support the declared version must report it as unsupported instead of guessing. + +## Directory Structure + +```shell +/ +└── / + ├── manifest.json ...dataset identity, versions, contents and checksums + ├── annotation/ ...schema tables in JSON format + ├── data/ ...sensor data, one file per channel + │ ├── / + │ │ └── .mp4 + │ └── / + │ └── .parquet + ├── input_bag/ ...original ROS bag file + └── map/ ...map files +``` + +See [Schema Tables](./table.md) for the tables and [Sensor Data](./data.md) for the sensor files. + +## manifest.json + +`manifest.json` is always JSON. It is written by the dataset producer and updated whenever a table or sensor file changes. + +```json +manifest { + "schema_version": -- Version of this specification, e.g. "1.0.0". Not the t4-devkit version. + "dataset_id": -- Dataset ID (= Web.Auto annotation_dataset_id). + "version": -- Dataset version (= Web.Auto version_id). + "created_at": -- Creation time of the dataset from the rosbag, ISO 8601. + "generators": <[Generator;N]> -- Every tool and library that produced this dataset. + "capabilities": -- What the dataset contains. + "annotation_specifications": <{str: AnnotationSpecification}> -- Annotation specification per task. + "tables": <{str: TableEntry}> -- Every table file, keyed by table name. + "assets": <[AssetEntry;N]> -- Every sensor, map and input bag file. + "provenance": -- Where the data came from. + "project_id": -- Web.Auto project ID. + "source_project_id": -- Web.Auto source project ID. + "is_synthetic": -- Whether the data is synthetic (= Web.Auto is_synthetic_data). + "is_discontinuous": -- Whether the recording has gaps (= Web.Auto). + "migrated_from_schema_version": -- Source version if the dataset was migrated, e.g. "0.x". `null` for datasets produced as 1.0. + "migration_notes": -- Notes on anything lost or changed during migration. +} +``` + +Only stable Web.Auto identifiers are stored. Values that change in Web.Auto after the dataset is produced (environment, map names, vehicle IDs) are not copied into the dataset. + +### `Generator` + +```json +Generator { + "name": -- Tool or library name, e.g. "t4-devkit", "tier4_perception_dataset". + "version": -- Version or commit hash. +} +``` + +### `Capabilities` + +```json +Capabilities { + "tasks": <[enum[detection3d, detection2d, segmentation2d, segmentation3d, keypoint, traffic_light, prediction, planning];N]> -- Tasks the dataset can be used for. + "modalities": { + "camera": <[str;N]> -- Camera channel names. + "lidar": <[str;N]> -- LiDAR channel names. + "radar": <[str;N]> -- Radar channel names. + } + "map": -- Whether a lanelet2 map is included. + "route": -- Whether the `route` table is filled. + "traffic_light": -- Whether the `traffic_light` table is filled. + "vehicle_state": -- Whether the `vehicle_state` table is filled. + "scenario": -- Whether the `scenario` table is filled. +} +``` + +### `AnnotationSpecification` + +```json +AnnotationSpecification { + "id": -- Annotation specification ID. + "version": -- Specification version, e.g. "4.1.9". `null` if unknown. +} +``` + +### `TableEntry` and `AssetEntry` + +```json +TableEntry { + "path": -- Path relative to the dataset root, e.g. "annotation/sample.json". + "sha256": -- SHA-256 of the file. +} + +AssetEntry { + "path": -- Path relative to the dataset root, e.g. "data/CAM_FRONT/CAM_FRONT.mp4". + "sha256": -- SHA-256 of the file. + "bytes": -- File size in bytes. +} +``` + +There is one `AssetEntry` per file, including each file in `input_bag/`. + +### `Provenance` + +```json +Provenance { + "rosbag": { + "log_file_ids": <[str;N]> -- Web.Auto log_file_id(s). Empty for force-pushed datasets. + "storage": -- rosbag storage format. + "record_start": -- Recording start, ISO 8601 UTC. + "record_end": -- Recording end, ISO 8601 UTC. + } + "topics": <{str: str}> -- ROS topic used for each channel, keyed by channel name. + "process_config_id": -- Web.Auto processing configuration ID. + "process_config_version_id": -- Web.Auto processing configuration version. +} +``` + +## Writing 1.0 Datasets + +`DatasetWriter` writes a 1.0 dataset folder. Data files go first, then the tables and `manifest.json`: + +```python +>>> from t4_devkit.schema.v1 import DatasetWriter +>>> writer = DatasetWriter("/") # must not exist yet +>>> path, frame_count = writer.write_lidar("LIDAR_TOP", frames, fields) # one Parquet file +>>> video = writer.write_camera("CAM_FRONT", images, frame_rate=10) # one AV1 video +>>> writer.copy("/map", "map") # files kept as they are +>>> manifest = writer.save(tables, manifest_info) # tables, then manifest.json +``` + +`save` lists every table and every file under `data/`, `map/` and `input_bag/` in `manifest.json`, with its hash. It computes `capabilities` from the tables, and adds t4-devkit to `generators`. `manifest_info` gives the other manifest fields (identity, provenance, migration notes). + +## Migrating 0.x Datasets + +[`t4migrate`](../../cli/t4migrate.md) writes a 0.x dataset as a new 1.0 dataset: the tables as loaded below, and one Parquet file or AV1 video per channel. In Python, `V0Migrator` does the same: + +```python +>>> from t4_devkit.schema.v1 import V0Migrator +>>> manifest = V0Migrator("<0.x dataset>", "out//").migrate("", ) +``` + +Metadata 0.x does not store (vehicle, map ids, provenance, routes, ...) can be added by the caller before writing: change records through `migrator.tables(...)`, and pass `manifest_fields` (e.g. `provenance`, `project_id`) to `migrate()`. + +## Loading 0.x Datasets + +`load_dataset()` reads a 0.x dataset (no `manifest.json`) by converting its tables to 1.0 records when loading, on a best-effort basis. Sensor data files stay as they are: one file per frame, and `sample_data.frame_index` is `null`. + +Every 0.x frame becomes a `sample`: the files of all channels with the same frame number (e.g. `00042.jpg` and `00042.pcd.bin`, as the 0.x converter names them) form one frame. 0.x samples, which are annotated frames only, keep their tokens; 0.x samples of the same frame (one per camera image in camera-only datasets) are merged into the earliest one. A frame with a 2D label is a key frame. + +Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 4: `equidistant`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC). + +Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). `instance.nbr_annotations` and its first / last annotation are computed over its 3D boxes and 2D labels; 0.x tools count them differently (some write 0 for 2D labels). Categories without `index` get their position in `category.json`, as the 0.x devkit reads them (all set or none). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`. + +`vehicle_state` rows get `source: vehicle_interface` (where the 0.x converter reads them) and shift `FORWARD`/`HIGH` becomes `DRIVE`; `additional_info.speed` is dropped and the velocities are `null`. 0.x `ego_pose.acceleration` is dropped too: it held raw IMU readings whose frame and gravity sign vary by vehicle, and 1.0 `acceleration` is gravity-free; `acceleration` and `imu` come from the backfill. The 0.x `traffic_light` table (`primitive_id`) and `traffic_light_instance_map` table (`traffic_light_linestring_id`, which wins if both list a lamp) link a lamp instance to its lanelet2 way (`type=traffic_light`) and have no states: the way becomes that instance's `map_element` (`type: way`), and the 1.0 `traffic_light` table stays empty. Source LiDARs of a concatenation have no frames of their own in 0.x: they take the concatenated channel's format and `fields`, and `frame_count` is the number of concatenated frames whose `.info` file lists points from them. + +1.0 requires some values that 0.x never stored. They are filled with these placeholders (`V0Converter.FALLBACKS`), which keep the dataset loadable but say nothing about it. In particular, `vehicle.model == "unknown"` means the vehicle dimensions are placeholders and must not be used, and `"unknown"` map IDs mean the map is not identified: + +| Table / block | Field | Placeholder | +| --------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `vehicle` | `model` | `"unknown"` | +| `vehicle` | `length`, `width`, `height`, `wheel_base` | `1.0` | +| `vehicle` | `name` | 0.x `log.vehicle`, `null` if empty | +| `map` | `map_id`, `version_id` | `"unknown"` | +| `map` | `source` | `"webauto"` | +| `ego_pose` | `source`, `frame` | `"localization"`, `"map"` | +| `sensor.camera` | `is_rectified` | `true` | +| `sensor.lidar` | `frame_id`, `is_undistorted`, `container.compression` | `"base_link"`, `true`, `"none"` | +| `sensor.lidar` | `fields` (`.bin`) | `x y z intensity channel return_type time_stamp`, cut to `num_pts_feats` (default 5), all `float32` | diff --git a/docs/schema/v1/requirement.md b/docs/schema/v1/requirement.md new file mode 100644 index 00000000..1b2c7a23 --- /dev/null +++ b/docs/schema/v1/requirement.md @@ -0,0 +1,189 @@ +# Dataset Requirements 1.0 + +The rules `t4sanity` checks on T4Dataset 1.0 datasets. The 0.x rules are in [Dataset Requirements](../requirement.md). + +## How Rules Apply + +- `t4sanity` reads the dataset version first (`manifest.json` present → 1.0) and runs only the rules for that version. The `Applies to` column says which. +- A rule ID keeps its meaning. Rules that mean the same for both versions keep their 0.x ID. Rules that change meaning get a new ID, numbered after the last 0.x ID of the group, so `--exclude` keeps working. +- The 0.x rules are not changed: 0.x datasets get exactly the same report as before 1.0. +- Rules within one record (field types, value ranges, `null` instead of `""`, the `sensor` block matching `modality`, the distortion model matching the number of coefficients, ...) are enforced when building the record, and reported by the format rules (`FMT`). The other groups check what involves several records or files. +- A 1.0 rule is skipped when the tables or files it needs could not be read, instead of failing; the format or structure rule that failed says why. +- Fixable 1.0 rules write tables, so they also update the hashes in `manifest.json`. + +## Structure (`STR`) + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | -------------------------------- | --------- | ------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `STR001` | `version-dir-presence` | `WARNING` | `N/A` | 0.x, 1.0 | `version/` directory exists under the dataset root directory. | +| `STR002` | `annotation-dir-presence` | `ERROR` | `N/A` | 0.x, 1.0 | `annotation/` directory exists under the dataset root directory. | +| `STR003` | `data-dir-presence` | `ERROR` | `N/A` | 0.x, 1.0 | `data/` directory exists under the dataset root directory. | +| `STR004` | `map-dir-presence` | `WARNING` | `N/A` | 0.x, 1.0 | `map/` directory exists under the dataset root directory. | +| `STR005` | `bag-dir-presence` | `WARNING` | `N/A` | 0.x, 1.0 | `input_bag/` directory exists under the dataset root directory. | +| `STR008` | `lanelet-file-presence` | `WARNING` | `N/A` | 0.x, 1.0 | `lanelet2_map.osm` file exists under the `map/` directory. | +| `STR009` | `pointcloud-map-dir-presence` | `WARNING` | `N/A` | 0.x, 1.0 | `pointcloud_map.pcd` directory exists under the `map/` directory. | +| `STR010` | `schema-version-supported` | `ERROR` | `N/A` | 1.0 | `manifest.json` declares a supported `schema_version` (major version 1). If not, no other 1.0 rule runs. | +| `STR011` | `manifest-files-presence` | `ERROR` | `N/A` | 1.0 | Every file listed in `manifest.json` (`tables` and `assets`) exists. | +| `STR012` | `manifest-files-checksum` | `ERROR` | `N/A` | 1.0 | Every listed file has the listed `sha256`, and every asset the listed `bytes`. Opt-in (`--checksum`), since it reads every file (videos, rosbags); without it, the rule reports a `WARNING` that hashes were not verified. | +| `STR013` | `manifest-files-complete` | `WARNING` | `N/A` | 1.0 | Every file under `annotation/`, `data/`, `map/` and `input_bag/` is listed in `manifest.json`. | +| `STR014` | `manifest-required-tables` | `ERROR` | `N/A` | 1.0 | `manifest.json` lists every required table: `sample`, `sample_data`, `ego_pose`, `sensor`, `scene`, `log`, `vehicle`. | +| `STR015` | `manifest-identity-matches-dirs` | `WARNING` | `N/A` | 1.0 | `manifest.json` `dataset_id` and `version` match the names of the dataset and version directories. | + +## Format (`FMT`) + +Each table file is a list of records that build into the 1.0 table class, with all its field and within-record rules. + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | ------------------------- | -------- | ------- | ---------- | ------------------------------------------------------------------------------------------ | +| `FMT020` | `manifest-field` | `ERROR` | `N/A` | 1.0 | All `manifest.json` fields are valid. | +| `FMT021` | `sample-field` | `ERROR` | `N/A` | 1.0 | All `Sample` records are valid. | +| `FMT022` | `sample-data-field` | `ERROR` | `N/A` | 1.0 | All `SampleData` records are valid. | +| `FMT023` | `ego-pose-field` | `ERROR` | `N/A` | 1.0 | All `EgoPose` records are valid. | +| `FMT024` | `sensor-field` | `ERROR` | `N/A` | 1.0 | All `Sensor` records are valid, including exactly one modality block, matching `modality`. | +| `FMT025` | `scene-field` | `ERROR` | `N/A` | 1.0 | All `Scene` records are valid. | +| `FMT026` | `log-field` | `ERROR` | `N/A` | 1.0 | All `Log` records are valid. | +| `FMT027` | `vehicle-field` | `ERROR` | `N/A` | 1.0 | All `Vehicle` records are valid. | +| `FMT028` | `map-field` | `ERROR` | `N/A` | 1.0 | All `Map` records are valid. | +| `FMT029` | `vehicle-state-field` | `ERROR` | `N/A` | 1.0 | All `VehicleState` records are valid. | +| `FMT030` | `category-field` | `ERROR` | `N/A` | 1.0 | All `Category` records are valid. | +| `FMT031` | `attribute-field` | `ERROR` | `N/A` | 1.0 | All `Attribute` records are valid. | +| `FMT032` | `visibility-field` | `ERROR` | `N/A` | 1.0 | All `Visibility` records are valid. | +| `FMT033` | `instance-field` | `ERROR` | `N/A` | 1.0 | All `Instance` records are valid. | +| `FMT034` | `sample-annotation-field` | `ERROR` | `N/A` | 1.0 | All `SampleAnnotation` records are valid. | +| `FMT035` | `object-ann-field` | `ERROR` | `N/A` | 1.0 | All `ObjectAnn` records are valid. | +| `FMT036` | `keypoint-field` | `ERROR` | `N/A` | 1.0 | All `Keypoint` records are valid. | +| `FMT037` | `traffic-light-field` | `ERROR` | `N/A` | 1.0 | All `TrafficLight` records are valid. | +| `FMT038` | `route-field` | `ERROR` | `N/A` | 1.0 | All `Route` records are valid. | +| `FMT039` | `scenario-field` | `ERROR` | `N/A` | 1.0 | All `Scenario` records are valid. | +| `FMT040` | `caption-field` | `ERROR` | `N/A` | 1.0 | All `Caption` records are valid. | + +## Schema Record (`REC`) + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | ------------------------------ | --------- | ------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REC001` | `scene-single` | `ERROR` | `N/A` | 0.x, 1.0 | `Scene` record is a single. | +| `REC002` | `sample-not-empty` | `ERROR` | `N/A` | 0.x, 1.0 | `Sample` record is not empty. | +| `REC003` | `sample-data-not-empty` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleData` record is not empty. | +| `REC004` | `ego-pose-not-empty` | `ERROR` | `N/A` | 0.x, 1.0 | `EgoPose` record is not empty. | +| `REC006` | `instance-not-empty` | `ERROR` | `N/A` | 0.x, 1.0 | `Instance` record is not empty if either `SampleAnnotation` or `ObjectAnnotation` is not empty. | +| `REC007` | `category-indices-consistent` | `ERROR` | `YES` | 0.x, 1.0 | `Category` records must either all have a unique `index` or all have a `null` index. | +| `REC008` | `sensor-not-empty` | `ERROR` | `N/A` | 1.0 | `Sensor` record is not empty. | +| `REC009` | `sensor-index-unique` | `ERROR` | `N/A` | 1.0 | `Sensor.index` values are unique. | +| `REC010` | `instance-index-unique` | `ERROR` | `N/A` | 1.0 | `Instance.index` values are unique where set. | +| `REC011` | `point-label-indices-presence` | `ERROR` | `N/A` | 1.0 | When a LiDAR file has `label` / `instance` columns, every `Category` has an `index`, and every `Instance` of an `is_thing` category an `index`. | +| `REC012` | `scene-sample-count` | `ERROR` | `YES` | 1.0 | `Scene.nbr_samples` equals the number of `Sample` records (frames) of the scene. | +| `REC013` | `instance-annotation-count` | `ERROR` | `YES` | 1.0 | `Instance.nbr_annotations` equals the number of its `SampleAnnotation` and `ObjectAnn` records, and its first / last annotation tokens point to its first / last one in time. | +| `REC014` | `thing-category-instance` | `ERROR` | `N/A` | 1.0 | `ObjectAnn.instance_token` is set exactly when its category has `is_thing`; `SampleAnnotation` and `Keypoint` instances belong to `is_thing` categories. | +| `REC015` | `orientation-number-allowed` | `ERROR` | `N/A` | 1.0 | `ObjectAnn.orientation` / `number` are set only for categories with `has_orientation` / `has_number`. | +| `REC016` | `manual-label-confidence` | `ERROR` | `N/A` | 1.0 | Labels with `annotation_type: manual` have `confidence: null`. | +| `REC017` | `automatic-label-provenance` | `WARNING` | `N/A` | 1.0 | Labels that are not `manual` have a `provenance`, and `auto` / `pseudo` labels a `confidence`. | +| `REC018` | `sample-frames` | `ERROR` | `N/A` | 1.0 | A `Sample` has at most one `SampleData` per channel; its `timestamp` is its LiDAR frame's (its earliest frame's if it has no LiDAR); `has_key_frame` is true exactly when one of its `SampleData` is a key frame. | +| `REC019` | `channel-frames` | `ERROR` | `N/A` | 1.0 | For a channel stored in one file, `frame_count` (`camera.video` / `lidar.container`) equals the number of its `SampleData` records, and their `frame_index` values are `0` to `frame_count - 1`, each once. | +| `REC020` | `sensor-storage-consistent` | `ERROR` | `N/A` | 1.0 | `SampleData.frame_index` is `null` exactly when its sensor stores one file per frame (`camera.video: null`, `lidar.container.path: null`), and `fileformat` matches the sensor's storage. | +| `REC021` | `timestamps-ordered` | `ERROR` | `N/A` | 1.0 | Timestamps increase along `next` (`Sample`, `SampleData`, `SampleAnnotation`), and `frame_index` follows timestamp order within a channel file. | +| `REC022` | `timestamps-microseconds` | `WARNING` | `N/A` | 1.0 | Timestamps are Unix microseconds: between 2000-01-01 and 2100-01-01 (catches seconds or nanoseconds). | +| `REC023` | `capabilities-present` | `ERROR` | `N/A` | 1.0 | Everything `manifest.json` `capabilities` declares is present: its camera / LiDAR channels have frames, `map` has a `Map` record, `route` / `traffic_light` / `vehicle_state` / `scenario` have records, and its tasks have their labels (see below). | +| `REC024` | `capabilities-complete` | `WARNING` | `N/A` | 1.0 | Everything present is declared in `capabilities`: the same checks as `REC023`, the other way round. | +| `REC025` | `no-placeholder-values` | `WARNING` | `N/A` | 1.0 | No placeholder values from loading or migrating 0.x: `Vehicle.model` and `Map.map_id` / `version_id` are not `"unknown"`. | +| `REC026` | `route-validity` | `ERROR` | `N/A` | 1.0 | `Route.valid_until` is after `valid_from` when set, and the routes of a log do not overlap in time. | +| `REC027` | `unit-quaternions` | `WARNING` | `N/A` | 1.0 | Rotations (`Sensor`, `EgoPose`, `SampleAnnotation`, `Route` poses) are unit quaternions. | +| `REC028` | `labels-on-key-frames` | `ERROR` | `N/A` | 1.0 | Labels sit on key frames: a `SampleAnnotation` on a `Sample` with `has_key_frame`, an `ObjectAnn` or `Keypoint` on a `SampleData` with `is_key_frame`. | +| `REC029` | `point-labels-reserved` | `ERROR` | `N/A` | 1.0 | When a LiDAR file has `label` / `instance` columns: the `Category` with `index` `0` is named `unpainted`, and each frame's label columns are `null` on all points or on none. | + +Tasks and the labels `REC023` / `REC024` expect: + +| Task | Expected labels | +| ---------------- | -------------------------------------------------------------- | +| `detection3d` | `SampleAnnotation` records | +| `detection2d` | `ObjectAnn` records of `is_thing` categories | +| `segmentation2d` | `ObjectAnn` records with a `mask` | +| `segmentation3d` | `label` / `instance` columns in a LiDAR file | +| `keypoint` | `Keypoint` records | +| `traffic_light` | `TrafficLight` records, or instances with a lamp `map_element` | +| `prediction` | Every `Sample` has a key frame (all frames annotated) | +| `planning` | `Scenario` records | + +## Reference (`REF`) + +`null` means "no link" in 1.0, where 0.x uses `""`: a reference rule does not apply to an empty link (the shared rules keep their 0.x wording). + +### Record Reference (A to B) + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | ------------------------------------- | -------- | ------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `REF001` | `scene-to-log` | `ERROR` | `N/A` | 0.x, 1.0 | `Scene.log_token` refers to `Log` record. | +| `REF002` | `scene-to-first-sample` | `ERROR` | `N/A` | 0.x, 1.0 | `Scene.first_sample_token` refers to `Sample` record. | +| `REF003` | `scene-to-last-sample` | `ERROR` | `N/A` | 0.x, 1.0 | `Scene.last_sample_token` refers to `Sample` record. | +| `REF004` | `sample-to-scene` | `ERROR` | `N/A` | 0.x, 1.0 | `Sample.scene_token` refers to `Scene` record. | +| `REF005` | `sample-data-to-sample` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleData.sample_token` refers to `Sample` record. | +| `REF006` | `sample-data-to-ego-pose` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleData.ego_pose_token` refers to `EgoPose` record. | +| `REF009` | `instance-to-category` | `ERROR` | `N/A` | 0.x, 1.0 | `Instance.category_token` refers to `Category` record. | +| `REF010` | `instance-to-first-sample-annotation` | `ERROR` | `N/A` | 0.x, 1.0 | `Instance.first_annotation_token` refers to `SampleAnnotation` or `ObjectAnn` record. | +| `REF011` | `instance-to-last-sample-annotation` | `ERROR` | `N/A` | 0.x, 1.0 | `Instance.last_annotation_token` refers to `SampleAnnotation` or `ObjectAnn` record. | +| `REF015` | `sample-data-to-sensor` | `ERROR` | `N/A` | 1.0 | `SampleData.sensor_token` refers to `Sensor` record. | +| `REF016` | `log-to-vehicle` | `ERROR` | `N/A` | 1.0 | `Log.vehicle_token` refers to `Vehicle` record. | +| `REF017` | `log-and-map` | `ERROR` | `N/A` | 1.0 | `Log.map_token` refers to `Map` record unless it is empty, `Map.log_tokens` refer to `Log` records, and both agree. | +| `REF018` | `sample-annotation-references` | `ERROR` | `N/A` | 1.0 | `SampleAnnotation` `sample_token`, `instance_token`, `attribute_tokens` and `visibility_token` (unless empty) refer to records. | +| `REF019` | `object-ann-references` | `ERROR` | `N/A` | 1.0 | `ObjectAnn` `sample_data_token`, `instance_token` (unless empty), `category_token` and `attribute_tokens` refer to records. | +| `REF020` | `keypoint-references` | `ERROR` | `N/A` | 1.0 | `Keypoint` `sample_data_token`, `instance_token` and `category_tokens` refer to records. | +| `REF021` | `traffic-light-references` | `ERROR` | `N/A` | 1.0 | `TrafficLight` `object_ann_token` and `instance_token` refer to records unless they are empty. | +| `REF022` | `route-references` | `ERROR` | `N/A` | 1.0 | `Route` `log_token` and `map_token` refer to records. | +| `REF023` | `scenario-references` | `ERROR` | `N/A` | 1.0 | `Scenario.log_token` refers to `Log` record. | +| `REF024` | `caption-references` | `ERROR` | `N/A` | 1.0 | `Caption` `log_token`, `sample_data_token`, `scenario_token` and `instance_tokens` refer to records unless they are empty. | +| `REF025` | `lidar-source-channels` | `ERROR` | `N/A` | 1.0 | `Lidar.source_channels` name LiDAR `Sensor` records that have no `SampleData`. | + +### Record Reference (A to A') + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | ----------------------------------- | -------- | ------- | ---------- | ------------------------------------------------------------------------------------------------ | +| `REF101` | `sample-next-to-another` | `ERROR` | `N/A` | 0.x, 1.0 | `Sample.next` refers to another one unless it is empty. | +| `REF102` | `sample-prev-to-another` | `ERROR` | `N/A` | 0.x, 1.0 | `Sample.prev` refers to another one unless it is empty. | +| `REF103` | `sample-annotation-next-to-another` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleAnnotation.next` refers to another one unless it is empty. | +| `REF104` | `sample-annotation-prev-to-another` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleAnnotation.prev` refers to another one unless it is empty. | +| `REF105` | `sample-data-next-to-another` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleData.next` refers to another one unless it is empty. | +| `REF106` | `sample-data-prev-to-another` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleData.prev` refers to another one unless it is empty. | +| `REF107` | `links-mutual` | `ERROR` | `N/A` | 1.0 | `next` and `prev` agree: `A.next` has `prev` = `A` (`Sample`, `SampleData`, `SampleAnnotation`). | +| `REF108` | `links-same-stream` | `ERROR` | `N/A` | 1.0 | `SampleData.next` is of the same channel, and `SampleAnnotation.next` of the same instance. | + +### File Reference + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | ------------------------------- | -------- | ------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REF201` | `sample-data-filename-presence` | `ERROR` | `N/A` | 0.x, 1.0 | `SampleData.filename` exists. | +| `REF204` | `sample-data-file-of-sensor` | `ERROR` | `N/A` | 1.0 | For a channel stored in one file, `SampleData.filename` is its sensor's `camera.video.path` / `lidar.container.path`. | +| `REF205` | `sensor-file-description` | `ERROR` | `N/A` | 1.0 | Each channel file matches its `sensor` row. Parquet: columns and dtypes are `lidar.fields`, one row group per frame. Video: codec, pixel format, bit depth, timescale and frame count are `camera.video`. Frame size is `camera.width` / `height`. | +| `REF206` | `lidar-num-points` | `ERROR` | `N/A` | 1.0 | `SampleData.num_points` equals the number of points in its frame, when set. | +| `REF207` | `map-file` | `ERROR` | `N/A` | 1.0 | `Map.filename` exists and has the SHA-256 `Map.sha256`. | + +### External Reference + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | -------------------------- | -------- | ------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `REF302` | `instance-to-map-element` | `ERROR` | `N/A` | 1.0 | `Instance.map_element` refers to an element of its `type` in the lanelet2 map; for traffic-light lamps, a way with `type=traffic_light`. | +| `REF303` | `traffic-light-to-lanelet` | `ERROR` | `N/A` | 1.0 | `TrafficLight.regulatory_element_id` is a traffic-light regulatory element of the lanelet2 map. | +| `REF304` | `route-to-lanelet` | `ERROR` | `N/A` | 1.0 | `Route` segment ids are lanelets of the lanelet2 map. | +| `REF305` | `lidar-source-index` | `ERROR` | `N/A` | 1.0 | In a merged LiDAR file, `source_index` values are the `Sensor.index` of the channel's `source_channels`. | +| `REF306` | `lidar-point-labels` | `ERROR` | `N/A` | 1.0 | In a LiDAR file, `label` values are `Category.index` values, and `instance` values are `Instance.index` values or `-1` (no object). | + +## TIER IV Instance (`TIV`) + +| ID | Name | Severity | Fixable | Applies to | Description | +| -------- | ------------ | -------- | ------- | ---------- | -------------------------------------------------- | +| `TIV001` | `load-tier4` | `ERROR` | `N/A` | 0.x, 1.0 | Ensure `T4Devkit` instance is loaded successfully. | + +## 0.x Rules Not Run on 1.0 + +| 0.x rule | Why | 1.0 rule | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------- | ---------------------------- | +| `STR006` `status-file-presence` | `manifest.json` replaces `status.json`. | `STR010`–`STR013` | +| `STR007` `schema-files-presence` | Required tables are listed in `manifest.json`. | `STR011`, `STR014` | +| `FMT001`–`FMT019` | The 1.0 tables are different classes; `calibrated_sensor`, `surface_ann` and `lidarseg` are gone. | `FMT020`–`FMT040` | +| `REC005` `calibrated-sensor-non-empty` | Merged into `sensor`. | `REC008` | +| `REF007` `sample-data-to-calibrated-sensor` | `SampleData` refers to `sensor` directly. | `REF015` | +| `REF008` `calibrated-sensor-to-sensor` | Merged into `sensor`. | `REF015` | +| `REF012` `lidarseg-to-sample-data` | Point-wise labels are LiDAR columns. | `REC011`, `REC029`, `REF306` | +| `REF203` `lidarseg-filename-presence` | Point-wise labels are LiDAR columns. | `REF205`, `REF306` | +| `REF013` `traffic-light-to-instance` | The 0.x lamp-to-way link is `Instance.map_element`; the 1.0 `traffic_light` table holds states. | `REF021`, `REF302` | +| `REF014` `traffic-light-to-lanelet` | Same as `REF013`. | `REF302`, `REF303` | +| `REF202` `sample-data-info-filename-presence` | Merged LiDAR sources are LiDAR columns instead of `.info` files. | `REF305` | +| `REF301` `pointcloud-metainfo-token-check` | Same as `REF202`. | `REF025`, `REF305` | diff --git a/docs/schema/v1/table.md b/docs/schema/v1/table.md new file mode 100644 index 00000000..4cf28a06 --- /dev/null +++ b/docs/schema/v1/table.md @@ -0,0 +1,550 @@ +# Schema Tables 1.0 + +Tables are stored as JSON files in `annotation/`, one file per table, each a list of records. Every table file is listed in `manifest.json` with its SHA-256. + +Types follow the notation in [Schema Tables](../table.md#type-definition). In addition: + +- Missing references are `null`, never an empty string `""`. +- Timestamps are Unix time in microseconds (`int`). +- Translations are `(x, y, z)` in meters and rotations are quaternions `(w, x, y, z)`. + +## Common Types + +### `AnnotationType` + +How a label was made: `enum["manual", "auto", "pseudo", "corrected", "interpolated", "synthetic"]`. + +| Value | Meaning | Example `provenance` | +| -------------- | -------------------------------------------------------------------------------------------------------- | --------------------------------- | +| `manual` | Made by a human annotator. | `"kognic"`, `"fastlabel"` | +| `auto` | Made by an auto-labeling model or pipeline, not reviewed by a human. | `"autolabel v2.3"` | +| `pseudo` | Taken from another system's output and used as a weak label, e.g. onboard perception or tracker results. | `"autoware multi_object_tracker"` | +| `corrected` | Made automatically (`auto`, `pseudo` or `interpolated`), then reviewed and corrected by a human. | `"autolabel v2.3 + kognic"` | +| `interpolated` | Computed between labeled frames, e.g. a box between two manual keyframes, not reviewed frame by frame. | `"t4-devkit interpolation"` | +| `synthetic` | Known by construction, from simulation or generated data (VLM, copy-paste). | `"carla 0.9.15"` | + +`confidence` is the producer's score for `auto` and `pseudo` labels, and `null` for `manual` labels. For other values it is optional. + +Every label table (`sample_annotation`, `object_ann`, `keypoint`, `traffic_light`, `scenario`, `caption`) has these three fields: + +```json +"annotation_type": -- How the label was made. +"provenance": -- Vendor, model name and version, or tool. +"confidence": -- Confidence in [0, 1]. `null` for manual labels. +``` + +They replace `automatic_annotation` and `autolabel_metadata` from 0.x. For 0.x readers, `automatic_annotation = (annotation_type != "manual")`. + +Labels sit on annotated frames (key frames): a `sample_annotation` on a `Sample` with `has_key_frame`, and an `object_ann` or `keypoint` on a `SampleData` with `is_key_frame`. Labeling a frame makes it a key frame, whatever the `annotation_type` (e.g. an interpolated box). An annotated frame may have no labels, e.g. when nothing was found or the labels are point-wise. + +## Required Tables + +### Sample + +- Filename: `sample.json` + +One frame of the recording: the observations of all channels at that moment, one `SampleData` per channel at most. Every frame is a sample, annotated or not; `has_key_frame` marks the annotated ones. In 0.x, only annotated frames were samples. + +```json +sample { + "token": -- Unique record identifier. + "timestamp": -- Unix timestamp in microseconds: the time of the sample's LiDAR frame, or of its earliest frame if it has no LiDAR. + "scene_token": -- Foreign key to the `Scene` table. + "has_key_frame": -- Whether any of its `SampleData` is a key frame (annotated). + "next": -- Foreign key to the next `Sample`. `null` for the last sample. + "prev": -- Foreign key to the previous `Sample`. `null` for the first sample. +} +``` + +### SampleData + +- Filename: `sample_data.json` + +One observation of one sensor channel, in one frame (`Sample`). Channel-level facts (calibration, image size, file encoding) are on the `Sensor` table. + +```json +sample_data { + "token": -- Unique record identifier. + "sample_token": -- Foreign key to the `Sample` (frame) this observation belongs to. + "ego_pose_token": -- Foreign key to the `EgoPose` table. + "sensor_token": -- Foreign key to the `Sensor` table. Replaces `calibrated_sensor_token`. + "filename": -- Path to the channel file (video or point cloud container), or to a per-frame file for datasets that keep 0.x files. + "frame_index": -- Index of the frame inside the channel file, starting at 0. `null` for per-frame files. + "fileformat": -- Format of the file. + "num_points": -- Number of points in the frame. LiDAR only. + "timestamp": -- Unix timestamp in microseconds (ROS header stamp of this observation). + "is_key_frame": -- Whether this frame was annotated. Per channel: in one sample, a camera may be annotated and another not. + "is_valid": -- `false` if the frame has a sensor drop or synchronization failure. + "next": -- Foreign key to the next `SampleData` of the same channel. + "prev": -- Foreign key to the previous `SampleData` of the same channel. +} +``` + +### EgoPose + +- Filename: `ego_pose.json` + +The ego vehicle's pose at a given time, in the map frame. + +```json +ego_pose { + "token": -- Unique record identifier. + "translation": <[float;3]> -- Position in meters. + "rotation": <[float;4]> -- Orientation as quaternion (w, x, y, z). + "timestamp": -- Unix timestamp in microseconds. + "twist": -- (vx, vy, vz, yaw_rate, pitch_rate, roll_rate) in the vehicle frame. + "acceleration": -- (ax, ay, az) in the vehicle frame [m/s²], without gravity: the localization estimate, what Autoware feeds the planner. + "imu": -- (ax, ay, az, wx, wy, wz): the raw IMU in the vehicle frame, specific force with gravity (az ≈ +9.8 m/s² at rest) and angular velocity [rad/s]. + "geocoordinate": -- (latitude, longitude, altitude). + "source": -- Where the pose came from. + "frame": -- Coordinate frame, e.g. "map". + "covariance": -- 6×6 pose covariance, row-major. +} +``` + +An `ego_pose` sits at a sample time. `twist`, `acceleration`, `imu`, `geocoordinate` and `covariance` hold the latest message at or before `timestamp`, best effort; the pose itself is interpolated at `timestamp`. + +### Sensor + +- Filename: `sensor.json` + +One row per sensor channel, with its calibration and how its data is stored. Merges `sensor` and `calibrated_sensor` from 0.x. A recalibration is a new row for the same channel. + +Exactly one modality block is present, matching `modality`. All fields inside a block are required when the block is present. + +```json +sensor { + "token": -- Unique record identifier. + "channel": -- Channel name, e.g. "CAM_FRONT", "LIDAR_CONCAT". + "index": -- Sensor index, 0–255, unique within the dataset. Value of the LiDAR `source_index` field for points from this sensor. + "modality": -- Sensor modality. + "translation": <[float;3]> -- Sensor position relative to base_link, in meters. + "rotation": <[float;4]> -- Sensor orientation relative to base_link, as quaternion (w, x, y, z). + "camera": -- Present when `modality` is "camera". + "lidar": -- Present when `modality` is "lidar". + "radar": -- Not defined in 1.0. + "hardware": -- Sensor hardware identity. +} +``` + +```json +Camera { + "width": -- Image width in pixels. + "height": -- Image height in pixels. + "intrinsic": <[[float;3];3]> -- Camera matrix. + "distortion": <[float;N]> -- Distortion coefficients, following `distortion_model`. + "distortion_model": -- From CameraInfo. + "is_rectified": -- Whether the stored images are already rectified. + "video": -- How the video is encoded. `null` for per-frame image files. +} + +Video { + "path": -- Path to the video file, e.g. "data/CAM_FRONT/CAM_FRONT.mp4". + "format": -- Container and codec. + "codec_profile": -- Encoder settings, e.g. "svt-av1 preset6 crf26 intra". + "pixel_format": -- e.g. "yuv420p full-range bt601". + "bit_depth": -- Bits per channel, 8 or 10. + "frame_count": -- Number of frames in the file. + "timescale": -- MP4 track timescale (ticks per second). Equal to the frame rate, so pts == frame_index. +} +``` + +```json +Lidar { + "container": { + "format": -- File format. + "path": -- Path to the file, e.g. "data/LIDAR_CONCAT/LIDAR_CONCAT.parquet". `null` for per-frame files. + "frame_count": -- Number of frames. + "compression": -- Compression profile, e.g. "zstd", or "none". + } + "fields": <[{"name": str, "dtype": str};N]> -- Point fields in column order, e.g. {"name": "x", "dtype": "float32"}. + "source_channels": <[str;N]> -- LiDAR channels merged into this one. Empty if not a concatenation. + "frame_id": -- Coordinate frame of the stored points, "base_link" by default. + "is_undistorted": -- Whether points are motion and scan compensated. +} +``` + +```json +Hardware { + "make": -- Manufacturer, e.g. "Hesai". + "model": -- Model, e.g. "OT128". + "serial_number": -- Serial number. + "firmware_version": -- Firmware version. +} +``` + +### Scene + +- Filename: `scene.json` + +Unchanged from 0.x. + +```json +scene { + "token": -- Unique record identifier. + "name": -- Name of the scene. + "description": -- Description of the scene. + "log_token": -- Foreign key to the `Log` table. + "nbr_samples": -- Number of samples (frames) in the scene. + "first_sample_token": -- Foreign key to the first `Sample`. + "last_sample_token": -- Foreign key to the last `Sample`. +} +``` + +### Log + +- Filename: `log.json` + +```json +log { + "token": -- Unique record identifier. + "logfile": -- Path to the log file. + "vehicle_token": -- Foreign key to the `Vehicle` table. + "data_captured": -- Capture time, ISO 8601. + "location": -- Location of the recording. + "map_token": -- Foreign key to the `Map` table. `null` if there is no map. +} +``` + +### Vehicle + +- Filename: `vehicle.json` + +Vehicle identity and dimensions. + +```json +vehicle { + "token": -- Unique record identifier. + "name": -- Vehicle name, e.g. "j6_gen2_04". + "model": -- Vehicle model. + "length": -- Length in meters. + "width": -- Width in meters. + "height": -- Height in meters. + "wheel_base": -- Wheel base in meters. + "rear_axle_to_center": -- Distance from the rear axle to the vehicle center, in meters. + "sensor_rig_id": -- Sensor rig ID. +} +``` + +## Optional Tables + +The following tables are optional. Which ones a dataset contains is declared in `manifest.json` (`capabilities`). + +### Map + +- Filename: `map.json` + +Map identity. Lane data stays in the lanelet2 file. + +```json +map { + "token": -- Unique record identifier. + "name": -- Map name. + "log_tokens": <[str;N]> -- Foreign keys to the `Log` table. + "format": -- Map format. + "filename": -- Path to the map file. + "sha256": -- SHA-256 of the map file. + "map_id": -- Web.Auto area_map_id. + "version_id": -- Web.Auto area_map_version_id. + "projector": { + "type": -- Map projection. + "mgrs_grid": -- MGRS grid. + "origin": { + "lat": -- Latitude in degrees. + "lon": -- Longitude in degrees. + "alt": -- Altitude in meters. + } + } + "source": -- Where the map came from. +} +``` + +### VehicleState + +- Filename: `vehicle_state.json` + +Measured vehicle signals at their native rate (not resampled). + +```json +vehicle_state { + "token": -- Unique record identifier. + "timestamp": -- Unix timestamp in microseconds. + "accel_pedal": -- Accelerator pedal position. + "brake_pedal": -- Brake pedal position. + "steer_pedal": -- Steering position. + "steering_tire_angle": -- Steering tire angle in radians. + "steering_wheel_angle": -- Steering wheel angle in radians. + "shift_state": -- Gear. + "indicators": -- Turn indicator and hazard state. + "longitudinal_velocity": -- Velocity in m/s. + "lateral_velocity": -- Velocity in m/s. + "heading_rate": -- Heading rate in rad/s. + "source": -- Where the signals came from. +} +``` + +`Indicators` is defined in [Schema Tables](../table.md#indicators). + +### Category + +- Filename: `category.json` + +```json +category { + "token": -- Unique record identifier. + "name": -- Category name. + "index": -- Value of the LiDAR `label` field for this category. Required when point-wise labels are present. `0` is reserved for the `unpainted` category. + "description": -- Description. + "is_thing": -- Whether annotations of this category are individual objects with an instance (car, pedestrian) rather than regions without one (road, sky). + "has_orientation": -- Whether annotations of this category may have `orientation`. + "has_number": -- Whether annotations of this category may have `number`. +} +``` + +### Attribute + +- Filename: `attribute.json` + +Unchanged from 0.x. + +```json +attribute { + "token": -- Unique record identifier. + "name": -- Attribute name. + "description": -- Description. +} +``` + +### Visibility + +- Filename: `visibility.json` + +Unknown visibility is `sample_annotation.visibility_token: null`; the 0.x level `unavailable` is dropped. + +```json +visibility { + "token": -- Unique record identifier. + "level": -- Visibility level. + "description": -- Description. +} +``` + +### Instance + +- Filename: `instance.json` + +An object identity. How each label was made is recorded on the label rows, not here. + +```json +instance { + "token": -- Unique record identifier. + "name": -- `::`. + "index": -- Value of the LiDAR `instance` field for this object, `0` to `32767` (`-1` in the field means no object). Unique within the dataset. Required when point-wise labels are present. + "category_token": -- Foreign key to the `Category` table. + "nbr_annotations": -- Number of annotations of this instance. + "first_annotation_token": -- Foreign key to the first `SampleAnnotation` or `ObjectAnn`. + "last_annotation_token": -- Foreign key to the last `SampleAnnotation` or `ObjectAnn`. + "map_element": { + "id": -- lanelet2 element ID: traffic-light lamp, sign, crosswalk, stop line, lane. + "type": -- lanelet2 element type. IDs are only unique within a type. + } +} +``` + +### SampleAnnotation + +- Filename: `sample_annotation.json` + +A 3D box in the map frame. + +```json +sample_annotation { + "token": -- Unique record identifier. + "sample_token": -- Foreign key to the `Sample` table. + "instance_token": -- Foreign key to the `Instance` table. + "attribute_tokens": <[str;N]> -- Foreign keys to the `Attribute` table. + "visibility_token": -- Foreign key to the `Visibility` table. `null` if unknown. + "translation": <[float;3]> -- Box center in meters. + "size": <[float;3]> -- (width, length, height) in meters. + "rotation": <[float;4]> -- Quaternion (w, x, y, z). + "velocity": -- (vx, vy, vz) in m/s. + "acceleration": -- (ax, ay, az) in m/s². + "num_lidar_pts": -- Number of LiDAR points in the box. `null` if not counted, e.g. no LiDAR data. + "num_radar_pts": -- Number of radar points in the box. `null` if not counted, e.g. no radar data. + "next": -- Foreign key to the next `SampleAnnotation` of the instance. + "prev": -- Foreign key to the previous `SampleAnnotation` of the instance. + "annotation_type": -- See [AnnotationType](#annotationtype). + "provenance": + "confidence": +} +``` + +### ObjectAnn + +- Filename: `object_ann.json` + +A 2D semantic/panoptic annotation in an image: a box with an optional mask. + +- For categories with `is_thing` (car, pedestrian), the mask is an instance mask and `instance_token` is set. +- For other categories (road, sidewalk, sky), the mask is a semantic mask of the region and `instance_token` is `null`. + +It replaces the 0.x `surface_ann` table. + +```json +object_ann { + "token": -- Unique record identifier. + "sample_data_token": -- Foreign key to the `SampleData` table. + "instance_token": -- Foreign key to the `Instance` table. `null` for categories without `is_thing`. + "category_token": -- Foreign key to the `Category` table. + "attribute_tokens": <[str;N]> -- Foreign keys to the `Attribute` table. + "bbox": <[float;4]> -- (xmin, ymin, xmax, ymax) in pixels. + "mask": -- Instance mask, or semantic mask for categories without `is_thing`. + "orientation": -- Arrow orientation in radians, for categories with `has_orientation`. + "number": -- Displayed digit, for categories with `has_number`. + "annotation_type": + "provenance": + "confidence": +} +``` + +`RLE` is defined in [Schema Tables](../table.md#rle). + +### Keypoint + +- Filename: `keypoint.json` + +```json +keypoint { + "token": -- Unique record identifier. + "sample_data_token": -- Foreign key to the `SampleData` table. + "instance_token": -- Foreign key to the `Instance` table. + "category_tokens": <[str;N]> -- Foreign keys to the `Category` table, one per keypoint. + "keypoints": <[[float;2];N]> -- (x, y) in pixels. + "num_keypoints": -- Number of keypoints. + "annotation_type": + "provenance": + "confidence": +} +``` + +### TrafficLight + +- Filename: `traffic_light.json` + +The state of one traffic light at one time, keyed by its lanelet2 regulatory element (= Autoware `TrafficLightGroup.traffic_light_group_id`). + +```json +traffic_light { + "token": -- Unique record identifier. + "timestamp": -- Unix timestamp in microseconds. + "regulatory_element_id": -- lanelet2 traffic light regulatory element ID. + "elements": <[TrafficLightElement;N]> -- Bulb states, as in Autoware `TrafficLightElement`. + "object_ann_token": -- Foreign key to the `ObjectAnn` the state was read from. + "instance_token": -- Foreign key to the lamp `Instance`, for states read from 2D labels. + "annotation_type": + "provenance": -- e.g. vendor, recognition model and version, "v2x". + "confidence": -- Confidence in [0, 1]. `null` for manual labels. +} + +TrafficLightElement { + "color": + "shape": + "status": + "confidence": -- Confidence in [0, 1]. +} +``` + +### Route + +- Filename: `route.json` + +The mission route, from Autoware `LaneletRoute`. A re-route adds a new row. + +```json +route { + "token": -- Unique record identifier. + "log_token": -- Foreign key to the `Log` table. + "map_token": -- Foreign key to the `Map` table. + "valid_from": -- Start of validity, Unix microseconds (inclusive). + "valid_until": -- End of validity, Unix microseconds (exclusive). `null` if valid until the end. + "route_uuid": -- `LaneletRoute.uuid`. + "start_pose": -- Start pose in the map frame. + "goal_pose": -- Goal pose in the map frame. + "segments": <[RouteSegment;N]> -- Route segments, in order. + "allow_modification": -- `LaneletRoute.allow_modification`. + "source": -- Where the route came from. + "provenance": -- Where the message was found, e.g. "own_bag", "route_json". +} + +RoutePose { + "translation": <[float;3]> -- (x, y, z) in meters. + "rotation": <[float;4]> -- Quaternion (w, x, y, z). +} + +RouteSegment { + "preferred_primitive_id": -- Preferred lanelet ID. + "primitive_ids": <[int;N]> -- All lanelet IDs of the segment. +} +``` + +### Scenario + +- Filename: `scenario.json` + +A tagged time interval, e.g. from CodSceneClassifier. + +```json +scenario { + "token": -- Unique record identifier. + "log_token": -- Foreign key to the `Log` table. + "start_timestamp": -- Start, Unix microseconds (inclusive). + "end_timestamp": -- End, Unix microseconds (exclusive). + "key_timestamp": -- Key moment, Unix microseconds. + "status": + "lateral_decision": + "longitudinal_decision": + "events": <[str;N]> + "scenery": -- Kept as given by the classifier. + "dynamic_entities": -- Kept as given by the classifier. + "justification": + "taxonomy": { + "id": -- Taxonomy ID, e.g. "cod-scene". + "version": -- Taxonomy version, e.g. "0.1.1". + } + "annotation_type": + "provenance": -- e.g. "cod-scene-classifier 1.0.2". + "confidence": -- Confidence in [0, 1]. `null` for manual labels. + "raw": -- Original record. +} +``` + +### Caption + +- Filename: `caption.json` + +Natural-language text about a time window, for VLM/VLA training. + +| `caption_type` | Meaning | +| ----------------- | ------------------------------------------------------ | +| `description` | Describes what happens in the window. | +| `reasoning` | Explains why the ego vehicle or others act as they do. | +| `instruction` | An instruction to the ego vehicle. | +| `question_answer` | A question about the window and its answer. | + +```json +caption { + "token": -- Unique record identifier. + "log_token": -- Foreign key to the `Log` table. + "start_timestamp": -- Start of the window, Unix microseconds. + "end_timestamp": -- End of the window. Equal to `start_timestamp` for a single moment. + "sample_data_token": -- Foreign key to the `SampleData` the caption describes, if it is about one image or sweep. + "scenario_token": -- Foreign key to a `Scenario`. + "instance_tokens": -- Foreign keys to the `Instance`s the text refers to. + "caption_type": -- Kind of text. + "text": -- The text. + "language": -- Language code, e.g. "en", "ja". + "annotation_type": -- VLM-generated captions are "auto". + "provenance": -- VLM name and version, prompt ID, annotator or vendor. + "confidence": +} +``` diff --git a/docs/tutorials/customize.md b/docs/tutorials/customize.md index 8b36e4e9..b56e8782 100644 --- a/docs/tutorials/customize.md +++ b/docs/tutorials/customize.md @@ -57,8 +57,8 @@ from __future__ import annotations from attrs import define, field -from t4_devkit.schema import SCHEMAS, SchemaName, SchemaBase -from t4_devkit.common import load_json +from t4_devkit.schema import SchemaBase +from t4_devkit.schema.v1 import SCHEMAS, SchemaName __all__ = ["CustomAttribute"] @@ -77,6 +77,8 @@ class CustomAttribute(SchemaBase): description: str | None = field(default=None) ``` +`T4Devkit` builds every table with the T4Dataset 1.0 registry (`t4_devkit.schema.v1.SCHEMAS`), for 1.0 datasets and for 0.x datasets (converted to 1.0 records when loading), so a custom class receives 1.0 records. + Note that `CustomAttribute` should be imported before instantiating `T4Devkit` class. Then modify `__init__.py` in order to import it automatically: diff --git a/docs/tutorials/initialize.md b/docs/tutorials/initialize.md index 96822783..b289a6dc 100644 --- a/docs/tutorials/initialize.md +++ b/docs/tutorials/initialize.md @@ -47,30 +47,33 @@ You can initialize a `T4Devkit` instance as follows: >>> t4 = T4Devkit("data/tier4/", verbose=True) ====== -Loading T4 tables in `annotation`... -Reverse indexing... -Done reverse indexing in 0.007 seconds. -====== -13 attribute -7 calibrated_sensor -8 category -2524 ego_pose -106 instance -1 log -1 map +Loading T4 tables... 72 sample -2390 sample_annotation 2524 sample_data -4 visibility +2524 ego_pose 7 sensor 1 scene +1 log +1 vehicle +1 map +0 vehicle_state +8 category +13 attribute +4 visibility +106 instance +2390 sample_annotation 1326 object_ann -0 surface_ann 0 keypoint +0 traffic_light +0 route +0 scenario +0 caption Done loading in 0.061 seconds. ====== ``` +Both T4Dataset 1.0 datasets (with `manifest.json`) and 0.x datasets load, as 1.0 records: 0.x tables are converted when loading (see [Loading 0.x Datasets](../schema/v1/index.md#loading-0x-datasets)). + Note that if you doesn't specify `revision` parameter in construction, it searches the latest version of the dataset. By specifying `revision=`, you can load the specific version of the dataset. @@ -90,7 +93,7 @@ By specifying `revision=`, you can load the specific version of the dat ### `sample` -`sample` is an annotated keyframe of a scene at a given timestamp. +`sample` is one frame of a scene: the observations of all channels at that moment. Every frame is a sample; `sample.has_key_frame` marks the annotated ones (key frames), which carry the labels. ```python >>> first_sample_token = my_scene.first_sample_token @@ -184,22 +187,12 @@ An `attribute` is a property of an instance that may change throughout different ### `sensor` T4 dataset consists of several type of sensors. -The supported sensor modalities and channels are defined in `t4_devkit/schema/tables/sensor.py`. +Each `sensor` row is one sensor with its calibration: `translation` and `rotation` with respect to the ego vehicle body frame, and a block for its modality (`camera` with intrinsics and image size, or `lidar` with its point fields). ```python >>> t4.sensor ``` -### `calibrated_sensor` - -`calibrated_sensor` consists of the definition of a calibration of a particular sensor based on a vehicle. - -```python ->>> t4.calibrated_sensor -``` - -Note that the `translation` and `rotation` parameters are given with respect to the ego vehicle body frame. - ### `ego_pose` `ego_pose` contains information about the `translation` and `rotation` of the ego vehicle, with respect to the global coordinate system. diff --git a/docs/tutorials/render.md b/docs/tutorials/render.md index c6286115..61c09cc4 100644 --- a/docs/tutorials/render.md +++ b/docs/tutorials/render.md @@ -36,7 +36,9 @@ If you want to visualize annotation results, `T4Devkit` supports some rendering >>> t4.render_pointcloud() ``` -If `SampleData.info_filename` points to a pointcloud metainfo JSON file, `T4Devkit.render_pointcloud()` loads it automatically. This allows rendering pointclouds with extended per-point features such as `return_type` or `timestamp`. When no metainfo file is available, the standard 5-feature layout is assumed. +Frames are read by their own format: point layouts come from the file (Parquet, PCD) or from `sensor.lidar.fields` (`.bin`; for 0.x datasets it is taken from the `.info` files when loading), and camera frames from per-frame images or from the channel's AV1 video (`sample_data.frame_index`). + +0.x LiDAR segmentation (`lidarseg`) is not rendered by `T4Devkit`. ![Render PointCloud GIF](../assets/render_pointcloud.gif) @@ -129,11 +131,11 @@ from t4_devkit.dataclass import LidarPointCloud from t4_devkit.viewer import PointCloudColorMode # Point cloud channel name >>> lidar_channel = "LIDAR_TOP" -# Load point cloud from file. -# Pass metainfo_filepath when the binary stores more than 5 float32 values per point. +# Load point cloud from file: one frame of a Parquet file, or a .pcd / .bin file. +# A .bin file has no header: pass its layout, e.g. from `sensor.lidar.fields`. >>> pointcloud = LidarPointCloud.from_file( -... , -... metainfo_filepath=, +... , +... frame_index=, ... ) >>> color_mode = PointCloudColorMode.DISTANCE >>> viewer.render_pointcloud(seconds, lidar_channel, pointcloud, color_mode) diff --git a/mkdocs.yaml b/mkdocs.yaml index 942f1899..233faf5d 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -7,10 +7,16 @@ nav: - t4-devkit: index.md - Installation: install.md - Dataset Schema: - - Home: schema/index.md - - Schema Tables: schema/table.md - - Sensor Data: schema/data.md - - Requirements: schema/requirement.md + - Schema 0.x: + - Home: schema/index.md + - Schema Tables: schema/table.md + - Sensor Data: schema/data.md + - Requirements: schema/requirement.md + - Schema 1.0: + - Home: schema/v1/index.md + - Schema Tables: schema/v1/table.md + - Sensor Data: schema/v1/data.md + - Requirements: schema/v1/requirement.md - Tutorials: - Initialization: tutorials/initialize.md - Visualization: tutorials/render.md @@ -20,6 +26,7 @@ nav: - Home: cli/index.md - t4viz: cli/t4viz.md - t4sanity: cli/t4sanity.md + - t4migrate: cli/t4migrate.md - API References: - t4_devkit.tier4: apis/tier4.md - t4_devkit.helper: apis/helper.md diff --git a/pyproject.toml b/pyproject.toml index 12a0b310..74b76535 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "returns>=0.26.0", "pyarrow<24.0.0", # NOTE: pyarrow==24.0.0 is only supported on macOS "pypcd4>=1.4.3,<2", + "av>=13.1", ] [dependency-groups] @@ -44,6 +45,7 @@ dev = [ [project.scripts] t4viz = "t4_devkit.cli.visualize:cli" t4sanity = "t4_devkit.cli.sanity:cli" +t4migrate = "t4_devkit.cli.migrate:cli" [tool.ruff] line-length = 100 diff --git a/t4_devkit/cli/migrate.py b/t4_devkit/cli/migrate.py new file mode 100644 index 00000000..122e5983 --- /dev/null +++ b/t4_devkit/cli/migrate.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import typer + +from t4_devkit.sanity import print_sanity_result, sanity_check +from t4_devkit.schema.v1 import V0Migrator + +from .version import version_callback + +cli = typer.Typer( + name="t4migrate", + no_args_is_help=True, + context_settings={"help_option_names": ["-h", "--help"]}, + pretty_exceptions_enable=False, +) + + +@cli.command() +def main( + version: bool = typer.Option( + False, + "--version", + "-v", + help="Show the application version and exit.", + callback=version_callback, + is_eager=True, + ), + data_root: str = typer.Argument( + ..., help="0.x dataset: the / directory with annotation/." + ), + output_dir: str = typer.Argument( + ..., + help="Parent directory: the dataset goes to //.", + ), + dataset_version: int = typer.Option( + ..., "-d", "--dataset-version", help="Version of the 1.0 dataset (given by Web.Auto)." + ), + dataset_id: str | None = typer.Option( + None, "--dataset-id", help="Dataset ID, if not the 0.x dataset's." + ), + keep_images: bool = typer.Option( + False, + "--keep-images", + help="Copy camera images as they are instead of encoding one AV1 video per channel.", + ), +) -> None: + source = Path(data_root) + if dataset_id is None: + # 0.x datasets live in / + dataset_id = source.parent.name if source.name.isdigit() else source.name + output_root = Path(output_dir) / dataset_id / str(dataset_version) + + migrator = V0Migrator(source, output_root) + manifest = migrator.migrate(dataset_id, dataset_version, keep_images=keep_images) + typer.echo(f"Migrated {source} to {output_root}") + for note in manifest.migration_notes or []: + typer.echo(f" - {note}") + + result = sanity_check(output_root.parent.as_posix(), revision=str(dataset_version)) + print_sanity_result(result) + sys.exit(0 if result.is_passed() else 1) diff --git a/t4_devkit/cli/sanity.py b/t4_devkit/cli/sanity.py index 3a06758d..3efc29c1 100644 --- a/t4_devkit/cli/sanity.py +++ b/t4_devkit/cli/sanity.py @@ -45,8 +45,17 @@ def main( fix: bool = typer.Option( False, "-f", "--fix", help="Attempt to fix the issues reported by the sanity check." ), + checksum: bool = typer.Option( + False, + "-c", + "--checksum", + help="Verify the SHA-256 of every file listed in manifest.json (1.0 datasets). " + "Reads every file, including videos and rosbags.", + ), ) -> None: - result = sanity_check(data_root=data_root, revision=revision, excludes=excludes, fix=fix) + result = sanity_check( + data_root=data_root, revision=revision, excludes=excludes, fix=fix, checksum=checksum + ) print_sanity_result(result, strict=strict) diff --git a/t4_devkit/cli/version.py b/t4_devkit/cli/version.py index 3080a24e..bb738e81 100644 --- a/t4_devkit/cli/version.py +++ b/t4_devkit/cli/version.py @@ -6,11 +6,10 @@ import typer from rich.console import Console -console = Console() - def version_callback(ctx: typer.Context, value: bool) -> None: if value: version = importlib.metadata.version("t4-devkit") + console = Console() console.print(f"[bold green]{ctx.info_name}[/bold green]: [cyan]{version}[/cyan]") raise typer.Exit() diff --git a/t4_devkit/common/io.py b/t4_devkit/common/io.py index 61966e49..fc7412b8 100644 --- a/t4_devkit/common/io.py +++ b/t4_devkit/common/io.py @@ -1,14 +1,38 @@ from __future__ import annotations +import hashlib import json -from typing import Any +import os +import threading +from fractions import Fraction +from functools import lru_cache +from typing import Any, Iterable, Mapping, Sequence +import av +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import pypcd4 +from PIL import Image from typing_extensions import TYPE_CHECKING if TYPE_CHECKING: from t4_devkit.typing import PathLike -__all__ = ("load_json", "save_json") +__all__ = ( + "load_json", + "save_json", + "load_pointcloud", + "save_pointcloud", + "load_image", + "save_video", + "file_sha256", + "V0_BIN_FIELDS", +) + +# 0.x `.bin` layout: this list cut to `num_pts_feats`, all float32 (as written by the converter). +# `.bin` stores no names; these are the 1.0 names (0.x calls `channel` "ring"). +V0_BIN_FIELDS = ("x", "y", "z", "intensity", "channel", "return_type", "time_stamp") def load_json(filename: PathLike) -> Any: @@ -34,3 +58,289 @@ def save_json(data: Any, filename: PathLike) -> None: """ with open(filename, "w") as f: json.dump(data, f, ensure_ascii=False, indent=4) + + +def file_sha256(filepath: PathLike) -> str: + """Return the SHA-256 of a file, read in 1 MiB chunks (videos and rosbags can be large). + + Args: + filepath (PathLike): File to hash. + + Returns: + The hex digest. + """ + digest = hashlib.sha256() + with open(os.fspath(filepath), "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_pointcloud( + filepath: PathLike, + *, + frame_index: int | None = None, + fields: Sequence[tuple[str, str]] | None = None, + columns: Sequence[str] | None = None, +) -> dict[str, np.ndarray]: + """Load one pointcloud frame as one array per point field, in the dtype stored in the file. + + The layout is read from the file itself (`.parquet`, `.pcd`), or from `fields` for `.bin`, + which has no header. Compression and encodings are read from the file. + + Args: + filepath (PathLike): `.parquet` file of a channel, or `.pcd` / `.bin` file of one frame. + frame_index (int | None, optional): Frame to read from a `.parquet` file (one row group per frame). + fields (Sequence[tuple[str, str]] | None, optional): (name, dtype) of each point field + of a `.bin` file, in order, e.g. from `sensor.lidar.fields`. + columns (Sequence[str] | None, optional): Fields to return; all fields if None. + + Returns: + Arrays of shape (N,) keyed by field name, in file order. A `.parquet` column that is + null on every point of the frame is left out. + """ + filepath = os.fspath(filepath) + if filepath.endswith(".parquet"): + if frame_index is None: + raise ValueError("frame_index is required for a .parquet file") + mtime = os.stat(filepath).st_mtime_ns + frame = _open_parquet(filepath, mtime, os.getpid()).read_row_group( + frame_index, columns=columns + ) + # a column null on every point (e.g. point labels of a frame without labels) is left out + return { + name: frame[name].to_numpy() + for name in frame.column_names + if frame.num_rows == 0 or frame[name].null_count < frame.num_rows + } + + if filepath.endswith(".pcd"): + data = pypcd4.PointCloud.from_path(filepath).pc_data + elif filepath.endswith(".bin"): + if fields is None: + raise ValueError("fields is required for a .bin file") + data = np.fromfile(filepath, dtype=np.dtype(list(fields))) + else: + raise ValueError(f"Unexpected filetype: {filepath}") + + names = data.dtype.names if columns is None else columns + return {name: data[name] for name in names} + + +def save_pointcloud( + filepath: PathLike, + frames: Iterable[Mapping[str, np.ndarray]], + fields: Sequence[tuple[str, str]], + *, + compression: str = "zstd", + compression_level: int | None = 1, +) -> int: + """Save the frames of a LiDAR channel as one Parquet file, one row group per frame. + + Columns are `fields`, in order. Encodings are chosen by dtype, the smallest in the 1.0 + benchmark that every tested reader (pyarrow, Polars, DuckDB) reads: floats BYTE_STREAM_SPLIT, + 8-bit integers dictionary, wider integers (e.g. timestamps) DELTA_BINARY_PACKED. Readers do + not depend on them. + + Args: + filepath (PathLike): `.parquet` file to write. + frames (Iterable[Mapping[str, np.ndarray]]): Arrays of shape (N,) keyed by field name, + one mapping per frame. A frame may have no points. A field that is missing or None + is null on every point of the frame, e.g. point labels of a frame without labels. + fields (Sequence[tuple[str, str]]): (name, dtype) of each column, e.g. from + `sensor.lidar.fields`. + compression (str, optional): Parquet compression codec. + compression_level (int | None, optional): Codec level; None for codecs without levels + (e.g. "snappy", "none"). + + Returns: + The number of frames written. + """ + schema = pa.schema([(name, pa.from_numpy_dtype(np.dtype(dtype))) for name, dtype in fields]) + encodings = {} + for field in schema: + if pa.types.is_floating(field.type): + encodings[field.name] = "BYTE_STREAM_SPLIT" + elif field.type.bit_width > 8: + encodings[field.name] = "DELTA_BINARY_PACKED" + num_frames = 0 + with pq.ParquetWriter( + os.fspath(filepath), + schema, + compression=compression, + compression_level=compression_level, + use_dictionary=[field.name for field in schema if field.name not in encodings], + column_encoding=encodings, + ) as writer: + for frame in frames: + values = [frame.get(name) for name, _ in fields] + num_points = next((len(v) for v in values if v is not None), None) + if num_points is None: + raise ValueError(f"Frame {num_frames} has no field with values") + columns = [ + pa.nulls(num_points, field.type) + if value is None + else pa.array(np.asarray(value, dtype=dtype), field.type) + for field, (_, dtype), value in zip(schema, fields, values) + ] + # one row group per frame, also for a frame without points + writer.write_table(pa.table(columns, schema=schema), row_group_size=max(num_points, 1)) + num_frames += 1 + return num_frames + + +@lru_cache(maxsize=32) +def _open_parquet(filepath: str, mtime: int, pid: int) -> pq.ParquetFile: + """Open a Parquet file once per process and file version, memory-mapped. + + Reopening per frame re-reads the footer. `mtime` and `pid` are only part of the cache key, + so a rewritten file or a forked worker gets its own handle. + """ + return pq.ParquetFile(filepath, memory_map=True) + + +def load_image(filepath: PathLike, *, frame_index: int | None = None) -> np.ndarray: + """Load one camera frame as an RGB image. + + With `frame_index`, the file is a video (one per channel) read with PyAV: container, codec, + pixel format, bit depth, color range and matrix, and timing all come from the file. + Without it, the file is one image per frame (`.jpg`, `.png`, ...) read with Pillow. + + Args: + filepath (PathLike): Video file of a channel, or image file of one frame. + frame_index (int | None, optional): Frame to read from a video file. + + Returns: + Image of shape (H, W, 3); uint8, or uint16 for video stored with more than 8 bits. + """ + filepath = os.fspath(filepath) + if frame_index is None: + with Image.open(filepath) as image: + return np.asarray(image.convert("RGB")) + + mtime = os.stat(filepath).st_mtime_ns + container = _open_video(filepath, mtime, os.getpid(), threading.get_ident()) + stream = container.streams.video[0] + # Constant frame rate: frame i is shown at i / rate seconds after the first frame. + pts = (stream.start_time or 0) + round(frame_index / (stream.average_rate * stream.time_base)) + # Seeks to the nearest keyframe at or before `pts` (the frame itself for intra-only video). + container.seek(pts, stream=stream) + for frame in container.decode(stream): + if frame.pts == pts: + # Decoded frames are YUV (e.g. yuv420p); convert to RGB at the stored bit depth so + # 8 bits per channel -> "rgb24" (uint8), more (e.g. 10-bit yuv420p10le) -> + # "rgb48le" (uint16). `components[0]` is the luma plane. + bit_depth = frame.format.components[0].bits + rgb_format = "rgb48le" if bit_depth > 8 else "rgb24" + return frame.reformat( + format=rgb_format, + src_colorspace=frame.colorspace, + src_color_range=frame.color_range, + ).to_ndarray() + if frame.pts > pts: + break + raise IndexError(f"Frame {frame_index} not found in {filepath}") + + +def save_video( + filepath: PathLike, + frames: Iterable[np.ndarray], + frame_rate: int, + *, + preset: int = 6, + crf: int = 26, +) -> dict[str, Any]: + """Save the frames of a camera channel as one AV1 MP4 video, with PyAV's SVT-AV1. + + Every frame is a keyframe. The track timescale is the frame rate, so a frame's pts is its + `frame_index`. Colors are YUV 4:2:0, full range, BT.601. + + Args: + filepath (PathLike): `.mp4` file to write. + frames (Iterable[np.ndarray]): RGB images of shape (H, W, 3), all the same size: uint8 for + 8-bit video, uint16 for 10-bit video. + frame_rate (int): Frames per second. + preset (int, optional): SVT-AV1 preset (speed vs size). + crf (int, optional): SVT-AV1 quality (lower is better). + + Returns: + The values of `sensor.camera.video`, except `path`. + """ + os.environ.setdefault("SVT_LOG", "1") + container = av.open( + os.fspath(filepath), + mode="w", + options={"movflags": "+faststart", "video_track_timescale": str(frame_rate)}, + ) + stream = None + num_frames = 0 + bit_depth, pixel_format = 8, "yuv420p" + with container: + for image in frames: + if stream is None: + # uint16 frames: 10-bit video + if image.dtype == np.uint16: + bit_depth, pixel_format = 10, "yuv420p10le" + stream = _add_video_stream(container, image, frame_rate, pixel_format, preset, crf) + rgb = av.VideoFrame.from_ndarray(image, format="rgb48le" if bit_depth > 8 else "rgb24") + frame = rgb.reformat( + format=pixel_format, + dst_colorspace=av.video.reformatter.Colorspace.ITU601, + dst_color_range=av.video.reformatter.ColorRange.JPEG, + ) + frame.pts = num_frames + container.mux(stream.encode(frame)) + num_frames += 1 + if stream is not None: + container.mux(stream.encode()) + return { + "format": "av1_mp4", + "codec_profile": f"svt-av1 preset{preset} crf{crf} intra", + "pixel_format": f"{pixel_format} full-range bt601", + "bit_depth": bit_depth, + "frame_count": num_frames, + "timescale": frame_rate, + } + + +def _add_video_stream( + container: av.container.OutputContainer, + image: np.ndarray, + frame_rate: int, + pixel_format: str, + preset: int, + crf: int, +) -> av.video.stream.VideoStream: + """Return the AV1 stream of a camera video, sized by its first image.""" + stream = container.add_stream("libsvtav1", rate=frame_rate) + stream.height, stream.width = image.shape[:2] + stream.pix_fmt = pixel_format + stream.time_base = Fraction(1, frame_rate) + context = stream.codec_context + context.gop_size = 1 # every frame a keyframe + context.options = { + "preset": str(preset), + "crf": str(crf), + # all intra, no scene detection or film grain, quiet + "svtav1-params": "pred-struct=0:scd=0:film-grain=0:fast-decode=2:lp=0:log-level=1", + } + # full range, BT.601 matrix, BT.709 primaries, sRGB transfer (FFmpeg enum values) + context.color_range = 2 + context.colorspace = 5 + context.color_primaries = 1 + context.color_trc = 13 + return stream + + +@lru_cache(maxsize=32) +def _open_video(filepath: str, mtime: int, pid: int, thread: int) -> av.container.InputContainer: + """Open a video file once per process, thread and file version. + + A container keeps decoding state, so threads do not share one. `mtime`, `pid` and `thread` + are only part of the cache key. + """ + container = av.open(filepath) + # One decoding thread: random access decodes one frame after each seek, where FFmpeg's + # default threading only adds latency (2880x1860 AV1: ~30 ms/frame default, ~20 ms with 1). + container.streams.video[0].codec_context.thread_count = 1 + return container diff --git a/t4_devkit/common/timestamp.py b/t4_devkit/common/timestamp.py index 281398de..27dca943 100644 --- a/t4_devkit/common/timestamp.py +++ b/t4_devkit/common/timestamp.py @@ -2,13 +2,14 @@ from typing import TYPE_CHECKING +from attrs import define from typing_extensions import deprecated if TYPE_CHECKING: from t4_devkit.typing import ScalarLike -__all__ = ["microseconds2seconds", "seconds2microseconds"] +__all__ = ["microseconds2seconds", "seconds2microseconds", "Stamp"] @deprecated("Please use `microseconds2seconds(...)` instead.") @@ -65,3 +66,25 @@ def seconds2microseconds(timestamp: ScalarLike) -> float: Timestamp in [us]. """ return 1e6 * timestamp + + +@define +class Stamp: + """A dataclass to represent timestamp. + + Attributes: + sec (int): Seconds. + nanosec (int): Nanoseconds. + """ + + sec: int + nanosec: int + + @property + def in_seconds(self) -> float: + """Convert timestamp to seconds as a float. + + Returns: + float: Timestamp in seconds. + """ + return self.sec + self.nanosec * 1e-9 diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index e354bb06..0c61f3c0 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -2,14 +2,15 @@ import os.path as osp import struct +import warnings from abc import abstractmethod -from typing import TYPE_CHECKING, ClassVar, TypeVar +from typing import TYPE_CHECKING, ClassVar, Sequence, TypeVar import numpy as np -import pypcd4 from attrs import define, field -from t4_devkit.common.io import load_json +from t4_devkit.common.io import V0_BIN_FIELDS, load_json, load_pointcloud +from t4_devkit.common.timestamp import Stamp if TYPE_CHECKING: from typing_extensions import Self @@ -28,28 +29,6 @@ ] -@define -class Stamp: - """A dataclass to represent timestamp. - - Attributes: - sec (int): Seconds. - nanosec (int): Nanoseconds. - """ - - sec: int - nanosec: int - - @property - def in_seconds(self) -> float: - """Convert timestamp to seconds as a float. - - Returns: - float: Timestamp in seconds. - """ - return self.sec + self.nanosec * 1e-9 - - @define class PointCloudSourceInfo: """A dataclass to represent pointcloud source information. @@ -261,30 +240,53 @@ def num_dims() -> int: return 4 @classmethod - def from_file(cls, filepath: str, metainfo_filepath: str | None = None) -> Self: - assert filepath.endswith((".bin", ".pcd")), f"Unexpected filetype: {filepath}" + def from_file( + cls, + filepath: str, + metainfo_filepath: str | None = None, + frame_index: int | None = None, + fields: Sequence[tuple[str, str]] | None = None, + ) -> Self: + """Create an object from a pointcloud file. + + Args: + filepath (str): `.parquet` file of a channel, or `.pcd` / `.bin` file of one frame. + metainfo_filepath (str | None, optional): Metainfo file of a 0.x frame. + frame_index (int | None, optional): Frame to read from a `.parquet` file. + fields (Sequence[tuple[str, str]] | None, optional): (name, dtype) of each point field + of a `.bin` file, e.g. from `sensor.lidar.fields`. + Returns: + Self instance. + """ metainfo = ( PointCloudMetainfo.from_file(metainfo_filepath) if metainfo_filepath is not None and osp.exists(metainfo_filepath) else None ) - if filepath.endswith(".bin"): + if fields is None and filepath.endswith(".bin"): num_pts_feats = getattr(metainfo, "num_pts_feats", 5) - scan = np.fromfile(filepath, dtype=np.float32) - points = scan.reshape((-1, num_pts_feats))[:, : cls.num_dims()] - else: - pcd = pypcd4.PointCloud.from_path(filepath) - points = pcd.numpy(("x", "y", "z", "intensity")).astype(np.float32) - - return cls(points.T, metainfo=metainfo) + if num_pts_feats > len(V0_BIN_FIELDS): + raise ValueError(f"Unknown 0.x point layout with {num_pts_feats} fields") + fields = [(name, "float32") for name in V0_BIN_FIELDS[:num_pts_feats]] + + xyzi = ("x", "y", "z", "intensity") # rows of `points` + columns = load_pointcloud(filepath, frame_index=frame_index, fields=fields, columns=xyzi) + points = np.empty((len(xyzi), len(columns["x"])), dtype=np.float32) + for i, name in enumerate(xyzi): + points[i] = columns[name] + return cls(points, metainfo=metainfo) @define class RadarPointCloud(PointCloud): """A dataclass to represent radar pointcloud. + Placeholder carried over from nuScenes: `from_file` reads the nuScenes radar PCD layout + (18 fields). tier4_perception_dataset stores radar as `RadarTracks` JSON, which it does + not read, and radar is not part of 1.0 yet. + Attributes: points (NDArrayFloat): Points matrix in the shape of (18, N). """ @@ -307,6 +309,10 @@ def from_file( ambig_states: list[int] | None = None, metainfo_filepath: str | None = None, ) -> Self: + warnings.warn( + "T4 radar is not implemented: RadarPointCloud reads the nuScenes radar PCD layout only.", + stacklevel=2, + ) assert filepath.endswith(".pcd"), f"Unexpected filetype: {filepath}" metadata: list[str] = [] @@ -399,6 +405,11 @@ def from_file( class SegmentationPointCloud(PointCloud): """A dataclass to represent segmentation pointcloud. + Deprecated: + Kept only for 0.x datasets, whose labels are a separate `.bin` file. In 1.0, labels are + columns of the LiDAR frame and come with `load_pointcloud()`; `T4Devkit` no longer + uses this class. + Attributes: points (NDArrayFloat): Points matrix in the shape of (4, N). labels (NDArrayU8): Label matrix. @@ -414,24 +425,9 @@ def num_dims() -> int: def from_file( cls, point_filepath: str, label_filepath: str, metainfo_filepath: str | None = None ) -> Self: - assert point_filepath.endswith((".bin", ".pcd")), f"Unexpected filetype: {point_filepath}" - - metainfo = ( - PointCloudMetainfo.from_file(metainfo_filepath) - if metainfo_filepath is not None and osp.exists(metainfo_filepath) - else None - ) - - if point_filepath.endswith(".bin"): - num_pts_feats = getattr(metainfo, "num_pts_feats", 5) - scan = np.fromfile(point_filepath, dtype=np.float32) - points = scan.reshape((-1, num_pts_feats))[:, : cls.num_dims()] - else: - pcd = pypcd4.PointCloud.from_path(point_filepath) - points = pcd.numpy(("x", "y", "z", "intensity")).astype(np.float32) - + pointcloud = LidarPointCloud.from_file(point_filepath, metainfo_filepath) labels = np.fromfile(label_filepath, dtype=np.uint8) - return cls(points.T, labels=labels, metainfo=metainfo) + return cls(pointcloud.points, labels=labels, metainfo=pointcloud.metainfo) def split_by_sensor(self) -> dict[str, PointCloud]: if self.metainfo is None: diff --git a/t4_devkit/helper/rendering.py b/t4_devkit/helper/rendering.py index ec3147d0..ec3b9124 100644 --- a/t4_devkit/helper/rendering.py +++ b/t4_devkit/helper/rendering.py @@ -11,12 +11,12 @@ import numpy as np import yaml +from t4_devkit.common.io import load_image from t4_devkit.common.timestamp import microseconds2seconds, seconds2microseconds -from t4_devkit.dataclass import LidarPointCloud, RadarPointCloud, SegmentationPointCloud +from t4_devkit.dataclass import LidarPointCloud, RadarPointCloud from t4_devkit.schema import SensorModality from t4_devkit.viewer import ( EntityPath, - PointCloudColorMode, RerunViewer, ViewerBuilder, format_entity, @@ -24,17 +24,14 @@ if TYPE_CHECKING: from t4_devkit import T4Devkit - from t4_devkit.schema import ( - CalibratedSensor, + from t4_devkit.schema.v1 import ( EgoPose, Instance, ObjectAnn, Sample, - SampleAnnotation, SampleData, Scene, Sensor, - SurfaceAnn, ) __all__ = ["RenderingHelper"] @@ -58,22 +55,32 @@ def __init__(self, t4: T4Devkit) -> None: t4 (T4Devkit): `T4Devkit` instance. """ self._t4 = t4 + # Class IDs for colors: `category.index`, or the list position when it is not set + # (1.0 sets it only for datasets with point-wise labels). self._label2id: dict[str, int] = { - category.name: category.index for category in self._t4.category - } - self._sample_data_to_lidarseg_filename: dict[str, str] = { - lidarseg.sample_data_token: lidarseg.filename for lidarseg in self._t4.lidarseg + category.name: position if category.index is None else category.index + for position, category in enumerate(self._t4.category) } self._executor = concurrent.futures.ThreadPoolExecutor() - def _has_lidarseg(self) -> bool: - """Return `True` if the dataset includes lidarseg.""" - return bool(self._sample_data_to_lidarseg_filename) + def _first_sample_data_tokens(self, modality: SensorModality) -> list[str]: + """Return the first frame of each channel of a modality, in sensor order. - def _find_lidarseg_file(self, sample_data_token: str) -> str | None: - """Try to find lidarseg filename from the sample data token.""" - return self._sample_data_to_lidarseg_filename.get(sample_data_token) + A recalibration adds a `sensor` row with the same channel, and source LiDARs of a merged + channel have no frames, so channels are taken once and sensors without frames skipped. + """ + first_tokens: dict[str, str] = {} + for sensor in self._t4.sensor: + if sensor.modality != modality or sensor.first_sd_token is None: + continue + current = first_tokens.get(sensor.channel) + if current is None or ( + self._t4.get("sample_data", sensor.first_sd_token).timestamp + < self._t4.get("sample_data", current).timestamp + ): + first_tokens[sensor.channel] = sensor.first_sd_token + return list(first_tokens.values()) def _init_viewer( self, @@ -91,9 +98,13 @@ def _init_viewer( render_ann (bool, optional): Indicates whether to render annotations. save_dir (str | None, optional): Directory path to save the rendering record. """ - cameras = [ - sensor.channel for sensor in self._t4.sensor if sensor.modality == SensorModality.CAMERA - ] + cameras = list( + dict.fromkeys( + sensor.channel + for sensor in self._t4.sensor + if sensor.modality == SensorModality.CAMERA + ) + ) builder = ( ViewerBuilder().with_spatial3d().with_spatial2d(cameras=cameras, contents=contents) @@ -130,7 +141,7 @@ def _load_contents(self, mode: RenderingMode, entity_child: str = "") -> list[st case RenderingMode.SCENE | RenderingMode.INSTANCE: # project 3D boxes/velocities/futures on image if there is no 2D annotation entity_root = format_entity(EntityPath.MAP, entity_child) - if len(self._t4.object_ann) == 0 and len(self._t4.surface_ann) == 0: + if len(self._t4.object_ann) == 0: contents = [ format_entity(entity_root, EntityPath.BOX), format_entity(entity_root, EntityPath.VELOCITY), @@ -161,21 +172,9 @@ def render_scene( show_map (bool, optional): Whether to render the map. """ # search first sample data tokens - first_lidar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.LIDAR - ] - first_radar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.RADAR - ] - first_camera_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.CAMERA - ] + first_lidar_tokens = self._first_sample_data_tokens(SensorModality.LIDAR) + first_radar_tokens = self._first_sample_data_tokens(SensorModality.RADAR) + first_camera_tokens = self._first_sample_data_tokens(SensorModality.CAMERA) app_id = f"scene@{self._t4.dataset_id}" contents = self._load_contents(RenderingMode.SCENE) @@ -188,18 +187,11 @@ def render_scene( first_sample: Sample = self._t4.get("sample", scene.first_sample_token) max_timestamp_us = first_sample.timestamp + seconds2microseconds(max_time_seconds) - pointcloud_color_mode = ( - PointCloudColorMode.SEGMENTATION - if self._has_lidarseg() - else PointCloudColorMode.DISTANCE - ) - futures = ( self._render_lidar_and_ego( viewer=viewer, first_lidar_tokens=first_lidar_tokens, max_timestamp_us=max_timestamp_us, - color_mode=pointcloud_color_mode, ) + self._render_radars( viewer=viewer, @@ -253,18 +245,16 @@ def render_instance( last_sample: Sample | None = None for token in instance_tokens: instance: Instance = self._t4.get("instance", token) - first_ann: SampleAnnotation = self._t4.get( - "sample_annotation", instance.first_annotation_token - ) - current_first_sample: Sample = self._t4.get("sample", first_ann.sample_token) - if first_sample is None or current_first_sample.timestamp < first_sample.timestamp: + current_first_sample = self._annotation_sample(instance.first_annotation_token) + if current_first_sample is not None and ( + first_sample is None or current_first_sample.timestamp < first_sample.timestamp + ): first_sample = current_first_sample - last_ann: SampleAnnotation = self._t4.get( - "sample_annotation", instance.last_annotation_token - ) - current_last_sample: Sample = self._t4.get("sample", last_ann.sample_token) - if last_sample is None or current_last_sample.timestamp > last_sample.timestamp: + current_last_sample = self._annotation_sample(instance.last_annotation_token) + if current_last_sample is not None and ( + last_sample is None or current_last_sample.timestamp > last_sample.timestamp + ): last_sample = current_last_sample if first_sample is None or last_sample is None: @@ -277,21 +267,9 @@ def render_instance( max_timestamp_us = last_sample.timestamp # search first sample data tokens - first_lidar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.LIDAR - ] - first_radar_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.RADAR - ] - first_camera_tokens: list[str] = [ - sensor.first_sd_token - for sensor in self._t4.sensor - if sensor.modality == SensorModality.CAMERA - ] + first_lidar_tokens = self._first_sample_data_tokens(SensorModality.LIDAR) + first_radar_tokens = self._first_sample_data_tokens(SensorModality.RADAR) + first_camera_tokens = self._first_sample_data_tokens(SensorModality.CAMERA) app_id = f"instance@{self._t4.dataset_id}" contents = self._load_contents(RenderingMode.INSTANCE) @@ -300,18 +278,11 @@ def render_instance( if show_map: self._try_render_map(viewer) - pointcloud_color_mode = ( - PointCloudColorMode.SEGMENTATION - if self._has_lidarseg() - else PointCloudColorMode.DISTANCE - ) - futures = ( self._render_lidar_and_ego( viewer=viewer, first_lidar_tokens=first_lidar_tokens, max_timestamp_us=max_timestamp_us, - color_mode=pointcloud_color_mode, ) + self._render_radars( viewer=viewer, @@ -363,16 +334,12 @@ def render_pointcloud( Add an option of rendering radar channels. """ # search first lidar sample data token - first_lidar_token: str | None = None - first_camera_tokens: list[str] = [] - for sensor in self._t4.sensor: - if sensor.modality == SensorModality.LIDAR: - first_lidar_token = sensor.first_sd_token - elif sensor.modality == SensorModality.CAMERA: - first_camera_tokens.append(sensor.first_sd_token) + first_lidar_tokens = self._first_sample_data_tokens(SensorModality.LIDAR) + first_camera_tokens = self._first_sample_data_tokens(SensorModality.CAMERA) - if first_lidar_token is None: + if not first_lidar_tokens: raise ValueError("There is no 3D pointcloud data.") + first_lidar_token = first_lidar_tokens[0] first_lidar_sample_data: SampleData = self._t4.get("sample_data", first_lidar_token) max_timestamp_us = first_lidar_sample_data.timestamp + seconds2microseconds( @@ -410,30 +377,20 @@ def _try_render_map(self, viewer: RerunViewer) -> None: def _render_sensor_calibration(self, viewer: RerunViewer, sample_data_token: str) -> None: sample_data: SampleData = self._t4.get("sample_data", sample_data_token) - calibration: CalibratedSensor = self._t4.get( - "calibrated_sensor", sample_data.calibrated_sensor_token - ) - sensor: Sensor = self._t4.get("sensor", calibration.sensor_token) - resolution = ( - (sample_data.width, sample_data.height) - if sensor.modality == SensorModality.CAMERA - else None - ) - viewer.render_calibration(sensor=sensor, calibration=calibration, resolution=resolution) + sensor: Sensor = self._t4.get("sensor", sample_data.sensor_token) + viewer.render_calibration(sensor) def _render_lidar_and_ego( self, viewer: RerunViewer, first_lidar_tokens: list[str], max_timestamp_us: float, - *, - color_mode: PointCloudColorMode = PointCloudColorMode.DISTANCE, ) -> list[Future]: def _render_single_lidar(first_lidar_token: str) -> None: self._render_sensor_calibration(viewer=viewer, sample_data_token=first_lidar_token) current_lidar_token = first_lidar_token - while current_lidar_token != "": + while current_lidar_token is not None: sample_data: SampleData = self._t4.get("sample_data", current_lidar_token) current_lidar_token = sample_data.next @@ -442,34 +399,19 @@ def _render_single_lidar(first_lidar_token: str) -> None: ego_pose: EgoPose = self._t4.get("ego_pose", sample_data.ego_pose_token) viewer.render_ego(ego_pose=ego_pose) - metainfo_filepath = ( - osp.join(self._t4.data_root, sample_data.info_filename) - if sample_data.info_filename - else None - ) - # render segmentation pointcloud if available, otherwise render raw pointcloud - if color_mode == PointCloudColorMode.SEGMENTATION: - label_filename = self._find_lidarseg_file(sample_data.token) - if label_filename is None: - continue - - pointcloud = SegmentationPointCloud.from_file( - point_filepath=osp.join(self._t4.data_root, sample_data.filename), - label_filepath=osp.join(self._t4.data_root, label_filename), - metainfo_filepath=metainfo_filepath, - ) - else: - pointcloud = LidarPointCloud.from_file( - osp.join(self._t4.data_root, sample_data.filename), - metainfo_filepath=metainfo_filepath, - ) + # the layout comes from the file, or from the sensor row for `.bin` + sensor: Sensor = self._t4.get("sensor", sample_data.sensor_token) + pointcloud = LidarPointCloud.from_file( + osp.join(self._t4.data_root, sample_data.filename), + frame_index=sample_data.frame_index, + fields=[(field.name, field.dtype) for field in sensor.lidar.fields], + ) viewer.render_pointcloud( seconds=microseconds2seconds(sample_data.timestamp), channel=sample_data.channel, pointcloud=pointcloud, - color_mode=color_mode, ) return [self._executor.submit(_render_single_lidar, token) for token in first_lidar_tokens] @@ -484,7 +426,7 @@ def _render_single_radar(first_radar_token: str) -> None: self._render_sensor_calibration(viewer=viewer, sample_data_token=first_radar_token) current_radar_token = first_radar_token - while current_radar_token != "": + while current_radar_token is not None: sample_data: SampleData = self._t4.get("sample_data", current_radar_token) if max_timestamp_us < sample_data.timestamp: @@ -513,16 +455,23 @@ def _render_single_camera(first_camera_token: str) -> None: self._render_sensor_calibration(viewer=viewer, sample_data_token=first_camera_token) current_camera_token = first_camera_token - while current_camera_token != "": + while current_camera_token is not None: sample_data: SampleData = self._t4.get("sample_data", current_camera_token) if max_timestamp_us < sample_data.timestamp: break + # An image file is passed as is (sent encoded); a video frame is decoded. + filepath = osp.join(self._t4.data_root, sample_data.filename) + image = ( + filepath + if sample_data.frame_index is None + else load_image(filepath, frame_index=sample_data.frame_index) + ) viewer.render_image( seconds=microseconds2seconds(sample_data.timestamp), camera=sample_data.channel, - image=osp.join(self._t4.data_root, sample_data.filename), + image=image, ) current_camera_token = sample_data.next @@ -541,16 +490,21 @@ def _render_annotation3ds( instance_tokens: Sequence[str] | None = None, ) -> None: current_sample_token = first_sample_token - while current_sample_token != "": + while current_sample_token is not None: sample: Sample = self._t4.get("sample", current_sample_token) if max_timestamp_us < sample.timestamp: break + # labels are on key frames only; logging none in between would clear them + if not sample.has_key_frame: + current_sample_token = sample.next + continue + if instance_tokens is not None: boxes = [] for ann_token in sample.ann_3ds: - ann: SampleAnnotation = self._t4.get("sample_annotation", ann_token) + ann = self._t4.get("sample_annotation", ann_token) if ann.instance_token in instance_tokens: boxes.append(self._t4.get_box3d(ann_token, future_seconds=future_seconds)) else: @@ -571,18 +525,23 @@ def _render_annotation2ds( instance_tokens: Sequence[str] | None = None, ) -> None: current_sample_token = first_sample_token - while current_sample_token != "": + while current_sample_token is not None: sample: Sample = self._t4.get("sample", current_sample_token) if max_timestamp_us < sample.timestamp: break + # labels are on key frames only; logging none in between would clear them + if not sample.has_key_frame: + current_sample_token = sample.next + continue + boxes = [] # For segmentation masks # TODO: declare specific class for segmentation mask in `dataclass` camera_masks: dict[str, dict[str, list]] = {} - # Object Annotation + # Object Annotation, including regions without an instance (road, sky) for obj_ann_token in sample.ann_2ds: obj_ann: ObjectAnn = self._t4.get("object_ann", obj_ann_token) box = self._t4.get_box2d(obj_ann_token) @@ -612,20 +571,6 @@ def _render_annotation2ds( # Render 2D box viewer.render_box2ds(microseconds2seconds(sample.timestamp), boxes) - if instance_tokens is None: - # Surface Annotation - for ann_token in sample.surface_anns: - surface_ann: SurfaceAnn = self._t4.get("surface_ann", ann_token) - sample_data: SampleData = self._t4.get( - "sample_data", surface_ann.sample_data_token - ) - camera_masks = _append_mask( - camera_masks, - camera=sample_data.channel, - ann=surface_ann, - class_id=self._label2id[surface_ann.category_name], - ) - # Render 2D segmentation image for camera, data in camera_masks.items(): viewer.render_segmentation2d( @@ -635,15 +580,30 @@ def _render_annotation2ds( # TODO: add support of rendering keypoints current_sample_token = sample.next + def _annotation_sample(self, annotation_token: str | None) -> Sample | None: + """Return the sample of an instance's first or last annotation, or None if there is none. + + The annotation is a `sample_annotation`, or an `object_ann` for instances labeled only in + 2D, whose sample is that of its frame. + """ + if annotation_token is None: + return None + try: + sample_token = self._t4.get("sample_annotation", annotation_token).sample_token + except KeyError: + object_ann: ObjectAnn = self._t4.get("object_ann", annotation_token) + sample_token = self._t4.get("sample_data", object_ann.sample_data_token).sample_token + return None if sample_token is None else self._t4.get("sample", sample_token) + def _append_mask( camera_masks: dict[str, dict[str, list]], camera: str, - ann: ObjectAnn | SurfaceAnn, + ann: ObjectAnn, class_id: int, uuid: str | None = None, ) -> dict[str, dict[str, list]]: - """Append segmentation mask data from `ObjectAnn/SurfaceAnn`. + """Append segmentation mask data from `ObjectAnn`. TODO: This function should be removed after declaring specific dataclass for 2D segmentation. @@ -651,7 +611,7 @@ def _append_mask( Args: camera_masks (dict[str, dict[str, list]]): Key-value data mapping camera name and mask data. camera (str): Name of camera channel. - ann (ObjectAnn | SurfaceAnn): Annotation object. + ann (ObjectAnn): Annotation object. class_id (int): Class ID. uuid (str | None, optional): Unique instance identifier. diff --git a/t4_devkit/helper/timeseries.py b/t4_devkit/helper/timeseries.py index 0559be4e..c4d1fe32 100644 --- a/t4_devkit/helper/timeseries.py +++ b/t4_devkit/helper/timeseries.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from t4_devkit import T4Devkit - from t4_devkit.schema import ObjectAnn, Sample, SampleAnnotation, SampleData + from t4_devkit.schema.v1 import ObjectAnn, Sample, SampleAnnotation, SampleData __all__ = ["TimeseriesHelper"] @@ -26,7 +26,7 @@ def __init__(self, t4: T4Devkit) -> None: self._sample_and_instance_to_ann3d: dict[tuple[str, str], str] = { (ann.sample_token, ann.instance_token): ann.token for ann in self._t4.sample_annotation } - self._sample_data_and_instance_to_ann2d: dict[tuple[str, str], str] = { + self._sample_data_and_instance_to_ann2d: dict[tuple[str, str | None], str] = { (ann.sample_data_token, ann.instance_token): ann.token for ann in self._t4.object_ann } @@ -54,7 +54,7 @@ def get_sample_annotations_until( anns: list[SampleAnnotation] = [] is_successor = seconds >= 0 current_sample_token = start_sample.next if is_successor else start_sample.prev - while current_sample_token != "": + while current_sample_token is not None: current_sample: Sample = self._t4.get("sample", current_sample_token) if abs(microseconds2seconds(current_sample.timestamp - start_sample.timestamp)) > abs( @@ -99,7 +99,7 @@ def get_object_anns_until( current_sample_data_token = ( start_sample_data.next if is_successor else start_sample_data.prev ) - while current_sample_data_token != "": + while current_sample_data_token is not None: current_sample_data: SampleData = self._t4.get("sample_data", current_sample_data_token) if abs( diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index e4a88d77..2d0038f6 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -5,18 +5,35 @@ from typing import TYPE_CHECKING, NewType from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful -from .result import make_report, make_skipped +from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS + +from .result import Reason, make_report, make_skipped if TYPE_CHECKING: + from t4_devkit.schema.v1 import SchemaName as V1SchemaName + from .context import SanityContext - from .result import Reason, Report + from .result import Report RuleID = NewType("RuleID", str) RuleName = NewType("RuleName", str) +def record_name(table: V1SchemaName) -> str: + """Return the class name of a 1.0 table's records (e.g. "SampleAnnotation"), for reasons.""" + return V1_SCHEMAS.get(table).__name__ + + +class SchemaVersion(str, Enum): + """Dataset schema versions a rule can apply to.""" + + V0 = "0.x" + V1 = "1.0" + + class Severity(str, Enum): """Severity levels for sanity checkers.""" @@ -33,12 +50,17 @@ def is_error(self) -> bool: class Checker(ABC): - """Base class for sanity checkers.""" + """Base class for sanity checkers. + + Attributes: + versions (tuple[SchemaVersion, ...]): Schema versions the rule applies to; 0.x by default. + """ id: RuleID name: RuleName description: str severity: Severity + versions: tuple[SchemaVersion, ...] = (SchemaVersion.V0,) def __call__(self, context: SanityContext, fix: bool = False) -> Report: """Run the checker and return a report. @@ -94,3 +116,38 @@ def fix(self, context: SanityContext) -> bool: True if the issue was fixed, False otherwise. """ return False + + +class ManifestChecker(Checker): + """Base class for 1.0 rules that read `manifest.json`: skipped when it cannot be read or is + not valid (reported by STR010 and FMT020).""" + + versions = (SchemaVersion.V1,) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match context.manifest: + case Some(_): + return Nothing + return Maybe.from_value(Reason("No valid 'manifest.json' (see STR010, FMT020)")) + + +class RecordsChecker(ManifestChecker): + """Base class for 1.0 rules on records: skipped unless every table they read is listed (or + optional) and builds (reported by STR014 and the format rules). + + Attributes: + tables (tuple[V1SchemaName, ...]): The tables the rule reads. + """ + + tables: tuple[V1SchemaName, ...] + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match super().can_skip(context): + case Some(reason): + return Some(reason) + invalid = [ + table.value for table in self.tables if not is_successful(context.load_records(table)) + ] + if invalid: + return Maybe.from_value(Reason(f"Tables not listed or not valid: {', '.join(invalid)}")) + return Nothing diff --git a/t4_devkit/sanity/context.py b/t4_devkit/sanity/context.py index 28537683..5ea15624 100644 --- a/t4_devkit/sanity/context.py +++ b/t4_devkit/sanity/context.py @@ -1,30 +1,50 @@ from __future__ import annotations +from functools import cached_property from pathlib import Path -from attrs import define -from returns.maybe import Maybe, Some +from attrs import define, field +from returns.converters import result_to_maybe +from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful +from returns.result import safe from typing_extensions import TYPE_CHECKING, Self from t4_devkit import DBMetadata +from t4_devkit.common.io import file_sha256 +from t4_devkit.lanelet import LaneletParser from t4_devkit.schema import SchemaName +from t4_devkit.schema.v1 import MANIFEST_FILENAME, SCHEMAS, Manifest +from t4_devkit.schema.v1 import SchemaName as V1SchemaName -from .safety import load_metadata_safe, save_json_safe +from .checker import SchemaVersion +from .safety import load_json_safe, load_manifest_safe, load_metadata_safe, save_json_safe if TYPE_CHECKING: from t4_devkit.typing import PathLike -@define +@define(slots=False) class SanityContext: + """What the checkers read: the dataset paths, and for 1.0 its manifest and tables, read once. + + Attributes: + metadata (Maybe[DBMetadata]): Dataset root, ID and version directory. + checksum (bool): Whether to verify the SHA-256 of the files listed in `manifest.json`. + """ + metadata: Maybe[DBMetadata] + checksum: bool = field(default=False, kw_only=True) + _tables: dict[V1SchemaName, Maybe[list[dict]]] = field(factory=dict, init=False, repr=False) + _records: dict[V1SchemaName, Maybe[list]] = field(factory=dict, init=False, repr=False) @classmethod - def from_path(cls, data_root: PathLike, revision: str | None = None) -> Self: + def from_path( + cls, data_root: PathLike, revision: str | None = None, *, checksum: bool = False + ) -> Self: metadata_result = load_metadata_safe(data_root, revision=revision) metadata = metadata_result.unwrap() if is_successful(metadata_result) else None - return cls(Maybe.from_optional(metadata)) + return cls(Maybe.from_optional(metadata), checksum=checksum) @property def data_root(self) -> Maybe[Path]: @@ -66,12 +86,95 @@ def status_json(self) -> Maybe[Path]: """Return the path to status JSON file, which is 'status.json'.""" return self.metadata.map(lambda m: Path(m.data_root).joinpath("status.json")) + @property + def manifest_json(self) -> Maybe[Path]: + """Return the path to `manifest.json`.""" + return self.metadata.map(lambda m: Path(m.data_root).joinpath(MANIFEST_FILENAME)) + + @property + def schema_version(self) -> SchemaVersion: + """Return 1.0 if `manifest.json` exists at the dataset root, 0.x otherwise.""" + match self.manifest_json: + case Some(x) if x.exists(): + return SchemaVersion.V1 + return SchemaVersion.V0 + + @cached_property + def raw_manifest(self) -> Maybe[dict]: + """Return `manifest.json` as read, or Nothing if it cannot be read.""" + match self.manifest_json: + case Some(x) if x.exists(): + return result_to_maybe(load_json_safe(x)) + return Nothing + + @cached_property + def manifest(self) -> Maybe[Manifest]: + """Return the manifest, or Nothing if it cannot be read or is not valid.""" + match self.raw_manifest: + case Some(data): + return result_to_maybe(load_manifest_safe(data)) + return Nothing + + def to_table_file(self, table: V1SchemaName) -> Maybe[Path]: + """Return the path of a 1.0 table file listed in `manifest.json`, or Nothing if unlisted.""" + match self.manifest, self.data_root: + case Some(manifest), Some(data_root) if table.value in manifest.tables: + return Some(data_root.joinpath(manifest.tables[table.value].path)) + return Nothing + + def load_table(self, table: V1SchemaName) -> Maybe[list[dict]]: + """Return the raw records of a 1.0 table as read (read once), or Nothing if unlisted or + unreadable.""" + if table not in self._tables: + match self.to_table_file(table): + case Some(path): + self._tables[table] = result_to_maybe(load_json_safe(path)) + case _: + self._tables[table] = Nothing + return self._tables[table] + + def load_records(self, table: V1SchemaName) -> Maybe[list]: + """Return the 1.0 records of a table (built once): empty for an unlisted optional table, + Nothing if the table is required and unlisted, unreadable, or has an invalid record + (reported by STR014 and the format rules).""" + if table not in self._records: + self._records[table] = self._build_records(table) + return self._records[table] + + @cached_property + def lanelet_map(self) -> Maybe[LaneletParser]: + """Return the lanelet2 map of the dataset (read once): the file of the first `Map` + record, or Nothing if there is none or it cannot be read.""" + match self.load_records(V1SchemaName.MAP), self.data_root: + case Some([map_record, *_]), Some(data_root): + return result_to_maybe(_parse_lanelet_safe(data_root.joinpath(map_record.filename))) + return Nothing + + def save_table(self, table: V1SchemaName, records: list[dict]) -> bool: + """Save the raw records of a 1.0 table, and its new SHA-256 in `manifest.json`. + + Returns: + True if both files were written. + """ + filepath = self.to_table_file(table).value_or(None) + if filepath is None or not is_successful(save_json_safe(records, filepath)): + return False + manifest = self.raw_manifest.unwrap() + manifest["tables"][table.value]["sha256"] = file_sha256(filepath) + saved = is_successful(save_json_safe(manifest, self.manifest_json.unwrap())) + # read again on next use + self._tables.pop(table, None) + self._records.pop(table, None) + return saved + def to_schema_file(self, schema: SchemaName) -> Maybe[Path]: """Convert schema name to file path, which is /annotation/.json.""" return self.annotation_dir.map(lambda ann: ann.joinpath(schema.filename)) def save_records(self, schema: SchemaName, records: list[dict]) -> bool: - """Save schema data to file.""" + """Save schema data to file (for 1.0, also its SHA-256 in `manifest.json`).""" + if self.schema_version == SchemaVersion.V1: + return self.save_table(V1SchemaName(schema.value), records) match self.to_schema_file(schema): case Some(filepath): result = save_json_safe(records, filepath) @@ -81,3 +184,24 @@ def save_records(self, schema: SchemaName, records: list[dict]) -> bool: print(f"Error saving schema: {result.failure()}") return False return False + + def _build_records(self, table: V1SchemaName) -> Maybe[list]: + match self.manifest: + case Some(manifest): + if table.value not in manifest.tables: + return Some([]) if table.is_optional() else Nothing + match self.load_table(table): + case Some(records): + return result_to_maybe(_build_safe(table, records)) + return Nothing + + +@safe +def _build_safe(table: V1SchemaName, records: list[dict]) -> list: + """Build the 1.0 records of a table; fails on the first invalid record.""" + return [SCHEMAS.get(table).from_dict(record) for record in records] + + +@safe +def _parse_lanelet_safe(path: Path) -> LaneletParser: + return LaneletParser(path.as_posix(), verbose=False) diff --git a/t4_devkit/sanity/format/__init__.py b/t4_devkit/sanity/format/__init__.py index d7af72ab..059af070 100644 --- a/t4_devkit/sanity/format/__init__.py +++ b/t4_devkit/sanity/format/__init__.py @@ -19,3 +19,24 @@ from .fmt017 import * # noqa from .fmt018 import * # noqa from .fmt019 import * # noqa +from .fmt020 import * # noqa +from .fmt021 import * # noqa +from .fmt022 import * # noqa +from .fmt023 import * # noqa +from .fmt024 import * # noqa +from .fmt025 import * # noqa +from .fmt026 import * # noqa +from .fmt027 import * # noqa +from .fmt028 import * # noqa +from .fmt029 import * # noqa +from .fmt030 import * # noqa +from .fmt031 import * # noqa +from .fmt032 import * # noqa +from .fmt033 import * # noqa +from .fmt034 import * # noqa +from .fmt035 import * # noqa +from .fmt036 import * # noqa +from .fmt037 import * # noqa +from .fmt038 import * # noqa +from .fmt039 import * # noqa +from .fmt040 import * # noqa diff --git a/t4_devkit/sanity/format/base.py b/t4_devkit/sanity/format/base.py index a776cff5..bde7afbd 100644 --- a/t4_devkit/sanity/format/base.py +++ b/t4_devkit/sanity/format/base.py @@ -6,8 +6,10 @@ from returns.pipeline import is_successful from t4_devkit.schema import SCHEMAS, SchemaName +from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS +from t4_devkit.schema.v1 import SchemaName as V1SchemaName -from ..checker import Checker +from ..checker import Checker, ManifestChecker from ..result import Reason from ..safety import load_json_safe, load_schema_safe @@ -55,3 +57,43 @@ def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason] | No if not is_successful(conversion): failures.append(Reason(f"[{schema.name}] {record['token']}: {conversion.failure()}")) return failures if failures else None + + +class TableFieldChecker(ManifestChecker): + """Base class for 1.0 format checkers: every record of a table builds into its 1.0 class. + + Attributes: + table (V1SchemaName): The 1.0 table to check. + """ + + table: V1SchemaName + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match super().can_skip(context): + case Some(reason): + return Some(reason) + if ( + not self.table.is_optional() + and self.table.value not in context.manifest.unwrap().tables + ): + return Maybe.from_value(Reason(f"'{self.table.value}' not listed (see STR014)")) + return Nothing + + def check(self, context: SanityContext) -> list[Reason] | None: + if self.table.value not in context.manifest.unwrap().tables: + return None # optional table not present + match context.load_table(self.table): + case Some(records): + return _build_v1_records(self.table, records) + return [Reason(f"'{context.to_table_file(self.table).unwrap()}' cannot be read")] + + +def _build_v1_records(table: V1SchemaName, records: list[dict]) -> list[Reason] | None: + module = V1_SCHEMAS.get(table) + failures = [] + for record in records: + conversion = load_schema_safe(module, record) + if not is_successful(conversion): + token = record.get("token") if isinstance(record, dict) else None + failures.append(Reason(f"[{table.name}] {token}: {conversion.failure()}")) + return failures if failures else None diff --git a/t4_devkit/sanity/format/fmt020.py b/t4_devkit/sanity/format/fmt020.py new file mode 100644 index 00000000..059b0dc6 --- /dev/null +++ b/t4_devkit/sanity/format/fmt020.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful + +from t4_devkit.schema.v1 import MANIFEST_FILENAME + +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity +from ..registry import CHECKERS +from ..result import Reason +from ..safety import load_manifest_safe + +if TYPE_CHECKING: + from ..context import SanityContext + +__all__ = ["FMT020"] + + +@CHECKERS.register() +class FMT020(Checker): + """A checker of FMT020.""" + + id = RuleID("FMT020") + name = RuleName("manifest-field") + severity = Severity.ERROR + description = "All 'manifest.json' fields are valid." + versions = (SchemaVersion.V1,) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match context.raw_manifest: + case Some(_): + return Nothing + return Maybe.from_value(Reason(f"'{MANIFEST_FILENAME}' cannot be read (see STR010)")) + + def check(self, context: SanityContext) -> list[Reason] | None: + result = load_manifest_safe(context.raw_manifest.unwrap()) + return None if is_successful(result) else [Reason(f"{result.failure()}")] diff --git a/t4_devkit/sanity/format/fmt021.py b/t4_devkit/sanity/format/fmt021.py new file mode 100644 index 00000000..d918b693 --- /dev/null +++ b/t4_devkit/sanity/format/fmt021.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT021"] + + +@CHECKERS.register() +class FMT021(TableFieldChecker): + """A checker of FMT021.""" + + id = RuleID("FMT021") + name = RuleName("sample-field") + severity = Severity.ERROR + description = "All 'Sample' records are valid." + table = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/format/fmt022.py b/t4_devkit/sanity/format/fmt022.py new file mode 100644 index 00000000..09cf313f --- /dev/null +++ b/t4_devkit/sanity/format/fmt022.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT022"] + + +@CHECKERS.register() +class FMT022(TableFieldChecker): + """A checker of FMT022.""" + + id = RuleID("FMT022") + name = RuleName("sample-data-field") + severity = Severity.ERROR + description = "All 'SampleData' records are valid." + table = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/format/fmt023.py b/t4_devkit/sanity/format/fmt023.py new file mode 100644 index 00000000..01ba7b4a --- /dev/null +++ b/t4_devkit/sanity/format/fmt023.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT023"] + + +@CHECKERS.register() +class FMT023(TableFieldChecker): + """A checker of FMT023.""" + + id = RuleID("FMT023") + name = RuleName("ego-pose-field") + severity = Severity.ERROR + description = "All 'EgoPose' records are valid." + table = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/format/fmt024.py b/t4_devkit/sanity/format/fmt024.py new file mode 100644 index 00000000..b8e8d774 --- /dev/null +++ b/t4_devkit/sanity/format/fmt024.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT024"] + + +@CHECKERS.register() +class FMT024(TableFieldChecker): + """A checker of FMT024.""" + + id = RuleID("FMT024") + name = RuleName("sensor-field") + severity = Severity.ERROR + description = ( + "All 'Sensor' records are valid, including exactly one modality block, matching 'modality'." + ) + table = SchemaName.SENSOR diff --git a/t4_devkit/sanity/format/fmt025.py b/t4_devkit/sanity/format/fmt025.py new file mode 100644 index 00000000..2355a9e6 --- /dev/null +++ b/t4_devkit/sanity/format/fmt025.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT025"] + + +@CHECKERS.register() +class FMT025(TableFieldChecker): + """A checker of FMT025.""" + + id = RuleID("FMT025") + name = RuleName("scene-field") + severity = Severity.ERROR + description = "All 'Scene' records are valid." + table = SchemaName.SCENE diff --git a/t4_devkit/sanity/format/fmt026.py b/t4_devkit/sanity/format/fmt026.py new file mode 100644 index 00000000..990883a0 --- /dev/null +++ b/t4_devkit/sanity/format/fmt026.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT026"] + + +@CHECKERS.register() +class FMT026(TableFieldChecker): + """A checker of FMT026.""" + + id = RuleID("FMT026") + name = RuleName("log-field") + severity = Severity.ERROR + description = "All 'Log' records are valid." + table = SchemaName.LOG diff --git a/t4_devkit/sanity/format/fmt027.py b/t4_devkit/sanity/format/fmt027.py new file mode 100644 index 00000000..99fd9ac6 --- /dev/null +++ b/t4_devkit/sanity/format/fmt027.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT027"] + + +@CHECKERS.register() +class FMT027(TableFieldChecker): + """A checker of FMT027.""" + + id = RuleID("FMT027") + name = RuleName("vehicle-field") + severity = Severity.ERROR + description = "All 'Vehicle' records are valid." + table = SchemaName.VEHICLE diff --git a/t4_devkit/sanity/format/fmt028.py b/t4_devkit/sanity/format/fmt028.py new file mode 100644 index 00000000..313becec --- /dev/null +++ b/t4_devkit/sanity/format/fmt028.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT028"] + + +@CHECKERS.register() +class FMT028(TableFieldChecker): + """A checker of FMT028.""" + + id = RuleID("FMT028") + name = RuleName("map-field") + severity = Severity.ERROR + description = "All 'Map' records are valid." + table = SchemaName.MAP diff --git a/t4_devkit/sanity/format/fmt029.py b/t4_devkit/sanity/format/fmt029.py new file mode 100644 index 00000000..9d2b9735 --- /dev/null +++ b/t4_devkit/sanity/format/fmt029.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT029"] + + +@CHECKERS.register() +class FMT029(TableFieldChecker): + """A checker of FMT029.""" + + id = RuleID("FMT029") + name = RuleName("vehicle-state-field") + severity = Severity.ERROR + description = "All 'VehicleState' records are valid." + table = SchemaName.VEHICLE_STATE diff --git a/t4_devkit/sanity/format/fmt030.py b/t4_devkit/sanity/format/fmt030.py new file mode 100644 index 00000000..8916fa0d --- /dev/null +++ b/t4_devkit/sanity/format/fmt030.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT030"] + + +@CHECKERS.register() +class FMT030(TableFieldChecker): + """A checker of FMT030.""" + + id = RuleID("FMT030") + name = RuleName("category-field") + severity = Severity.ERROR + description = "All 'Category' records are valid." + table = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/format/fmt031.py b/t4_devkit/sanity/format/fmt031.py new file mode 100644 index 00000000..4d962100 --- /dev/null +++ b/t4_devkit/sanity/format/fmt031.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT031"] + + +@CHECKERS.register() +class FMT031(TableFieldChecker): + """A checker of FMT031.""" + + id = RuleID("FMT031") + name = RuleName("attribute-field") + severity = Severity.ERROR + description = "All 'Attribute' records are valid." + table = SchemaName.ATTRIBUTE diff --git a/t4_devkit/sanity/format/fmt032.py b/t4_devkit/sanity/format/fmt032.py new file mode 100644 index 00000000..67f8af2f --- /dev/null +++ b/t4_devkit/sanity/format/fmt032.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT032"] + + +@CHECKERS.register() +class FMT032(TableFieldChecker): + """A checker of FMT032.""" + + id = RuleID("FMT032") + name = RuleName("visibility-field") + severity = Severity.ERROR + description = "All 'Visibility' records are valid." + table = SchemaName.VISIBILITY diff --git a/t4_devkit/sanity/format/fmt033.py b/t4_devkit/sanity/format/fmt033.py new file mode 100644 index 00000000..1666c7e7 --- /dev/null +++ b/t4_devkit/sanity/format/fmt033.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT033"] + + +@CHECKERS.register() +class FMT033(TableFieldChecker): + """A checker of FMT033.""" + + id = RuleID("FMT033") + name = RuleName("instance-field") + severity = Severity.ERROR + description = "All 'Instance' records are valid." + table = SchemaName.INSTANCE diff --git a/t4_devkit/sanity/format/fmt034.py b/t4_devkit/sanity/format/fmt034.py new file mode 100644 index 00000000..5aca2a34 --- /dev/null +++ b/t4_devkit/sanity/format/fmt034.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT034"] + + +@CHECKERS.register() +class FMT034(TableFieldChecker): + """A checker of FMT034.""" + + id = RuleID("FMT034") + name = RuleName("sample-annotation-field") + severity = Severity.ERROR + description = "All 'SampleAnnotation' records are valid." + table = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/format/fmt035.py b/t4_devkit/sanity/format/fmt035.py new file mode 100644 index 00000000..28e323f7 --- /dev/null +++ b/t4_devkit/sanity/format/fmt035.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT035"] + + +@CHECKERS.register() +class FMT035(TableFieldChecker): + """A checker of FMT035.""" + + id = RuleID("FMT035") + name = RuleName("object-ann-field") + severity = Severity.ERROR + description = "All 'ObjectAnn' records are valid." + table = SchemaName.OBJECT_ANN diff --git a/t4_devkit/sanity/format/fmt036.py b/t4_devkit/sanity/format/fmt036.py new file mode 100644 index 00000000..1cf64d8e --- /dev/null +++ b/t4_devkit/sanity/format/fmt036.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT036"] + + +@CHECKERS.register() +class FMT036(TableFieldChecker): + """A checker of FMT036.""" + + id = RuleID("FMT036") + name = RuleName("keypoint-field") + severity = Severity.ERROR + description = "All 'Keypoint' records are valid." + table = SchemaName.KEYPOINT diff --git a/t4_devkit/sanity/format/fmt037.py b/t4_devkit/sanity/format/fmt037.py new file mode 100644 index 00000000..1bf823b0 --- /dev/null +++ b/t4_devkit/sanity/format/fmt037.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT037"] + + +@CHECKERS.register() +class FMT037(TableFieldChecker): + """A checker of FMT037.""" + + id = RuleID("FMT037") + name = RuleName("traffic-light-field") + severity = Severity.ERROR + description = "All 'TrafficLight' records are valid." + table = SchemaName.TRAFFIC_LIGHT diff --git a/t4_devkit/sanity/format/fmt038.py b/t4_devkit/sanity/format/fmt038.py new file mode 100644 index 00000000..57faa26a --- /dev/null +++ b/t4_devkit/sanity/format/fmt038.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT038"] + + +@CHECKERS.register() +class FMT038(TableFieldChecker): + """A checker of FMT038.""" + + id = RuleID("FMT038") + name = RuleName("route-field") + severity = Severity.ERROR + description = "All 'Route' records are valid." + table = SchemaName.ROUTE diff --git a/t4_devkit/sanity/format/fmt039.py b/t4_devkit/sanity/format/fmt039.py new file mode 100644 index 00000000..a596e909 --- /dev/null +++ b/t4_devkit/sanity/format/fmt039.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT039"] + + +@CHECKERS.register() +class FMT039(TableFieldChecker): + """A checker of FMT039.""" + + id = RuleID("FMT039") + name = RuleName("scenario-field") + severity = Severity.ERROR + description = "All 'Scenario' records are valid." + table = SchemaName.SCENARIO diff --git a/t4_devkit/sanity/format/fmt040.py b/t4_devkit/sanity/format/fmt040.py new file mode 100644 index 00000000..348443ad --- /dev/null +++ b/t4_devkit/sanity/format/fmt040.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT040"] + + +@CHECKERS.register() +class FMT040(TableFieldChecker): + """A checker of FMT040.""" + + id = RuleID("FMT040") + name = RuleName("caption-field") + severity = Severity.ERROR + description = "All 'Caption' records are valid." + table = SchemaName.CAPTION diff --git a/t4_devkit/sanity/record/__init__.py b/t4_devkit/sanity/record/__init__.py index 92398fa0..3af0f9fd 100644 --- a/t4_devkit/sanity/record/__init__.py +++ b/t4_devkit/sanity/record/__init__.py @@ -7,3 +7,25 @@ from .rec005 import * # noqa from .rec006 import * # noqa from .rec007 import * # noqa +from .rec008 import * # noqa +from .rec009 import * # noqa +from .rec010 import * # noqa +from .rec011 import * # noqa +from .rec012 import * # noqa +from .rec013 import * # noqa +from .rec014 import * # noqa +from .rec015 import * # noqa +from .rec016 import * # noqa +from .rec017 import * # noqa +from .rec018 import * # noqa +from .rec019 import * # noqa +from .rec020 import * # noqa +from .rec021 import * # noqa +from .rec022 import * # noqa +from .rec023 import * # noqa +from .rec024 import * # noqa +from .rec025 import * # noqa +from .rec026 import * # noqa +from .rec027 import * # noqa +from .rec028 import * # noqa +from .rec029 import * # noqa diff --git a/t4_devkit/sanity/record/base.py b/t4_devkit/sanity/record/base.py index 57984ea9..62540fb5 100644 --- a/t4_devkit/sanity/record/base.py +++ b/t4_devkit/sanity/record/base.py @@ -4,14 +4,19 @@ from typing import TYPE_CHECKING from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful from t4_devkit.schema import SchemaName +from t4_devkit.schema.v1 import Capabilities, FileFormat, LidarContainerFormat +from t4_devkit.schema.v1 import SchemaName as V1SchemaName from ..checker import Checker from ..result import Reason from ..safety import load_json_safe if TYPE_CHECKING: + from t4_devkit.schema.v1 import Sensor + from ..context import SanityContext @@ -33,6 +38,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case Some(x): if not x.exists(): return Maybe.from_value(Reason(f"Missing {self.schema.filename}")) + elif not is_successful(load_json_safe(x)): + return Maybe.from_value(Reason(f"Cannot read {self.schema.filename}")) else: return Nothing case _: @@ -54,3 +61,52 @@ def check_count(self, records: list[dict]) -> list[Reason] | None: A list of reasons for any issues found, otherwise None. """ pass + + +def channel_file(sensor: Sensor) -> tuple[str, int] | None: + """Return the file and frame count of a channel stored in one file, or None for one file per + frame.""" + if sensor.camera is not None and sensor.camera.video is not None: + return sensor.camera.video.path, sensor.camera.video.frame_count + if sensor.lidar is not None and sensor.lidar.container.path is not None: + return sensor.lidar.container.path, sensor.lidar.container.frame_count + return None + + +def frame_formats(sensor: Sensor) -> set[str]: + """Return the `sample_data.fileformat` values a sensor's storage allows (any for radar, + undefined in 1.0).""" + if sensor.camera is not None: + if sensor.camera.video is not None: + return {FileFormat.AV1_MP4.value} + return {FileFormat.JPG.value, FileFormat.PNG.value} + if sensor.lidar is not None: + lidar_formats = { + LidarContainerFormat.PARQUET: {FileFormat.PARQUET.value}, + LidarContainerFormat.PCD: {FileFormat.PCD.value}, + LidarContainerFormat.BIN: {FileFormat.BIN.value, FileFormat.PCDBIN.value}, + } + return lidar_formats[sensor.lidar.container.format] + return {fileformat.value for fileformat in FileFormat} + + +def capability_items(capabilities: Capabilities) -> set[str]: + """Return capabilities as comparable items, e.g. "camera:CAM_FRONT", "map", "task:keypoint".""" + items = { + f"{modality}:{channel}" + for modality in ("camera", "lidar", "radar") + for channel in getattr(capabilities.modalities, modality) + } + items |= { + name + for name in ("map", "route", "traffic_light", "vehicle_state", "scenario") + if getattr(capabilities, name) + } + items |= {f"task:{task.value}" for task in capabilities.tasks} + return items + + +def present_capabilities(context: SanityContext, tables: tuple[V1SchemaName, ...]) -> set[str]: + """Return what the dataset's tables contain, as `capability_items`.""" + records = {table: context.load_records(table).unwrap() for table in tables} + return capability_items(Capabilities.from_tables(records)) diff --git a/t4_devkit/sanity/record/rec001.py b/t4_devkit/sanity/record/rec001.py index dff52a88..55dfeb21 100644 --- a/t4_devkit/sanity/record/rec001.py +++ b/t4_devkit/sanity/record/rec001.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC001(RecordCountChecker): id = RuleID("REC001") name = RuleName("scene-single") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene' record is a single." schema = SchemaName.SCENE diff --git a/t4_devkit/sanity/record/rec002.py b/t4_devkit/sanity/record/rec002.py index 48c66833..503dfa82 100644 --- a/t4_devkit/sanity/record/rec002.py +++ b/t4_devkit/sanity/record/rec002.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC002(RecordCountChecker): id = RuleID("REC002") name = RuleName("sample-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample' record is not empty." schema = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/record/rec003.py b/t4_devkit/sanity/record/rec003.py index 748f018d..e1b8f881 100644 --- a/t4_devkit/sanity/record/rec003.py +++ b/t4_devkit/sanity/record/rec003.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC003(RecordCountChecker): id = RuleID("REC003") name = RuleName("sample-data-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData' record is not empty." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/record/rec004.py b/t4_devkit/sanity/record/rec004.py index 93ac1cad..bdb71612 100644 --- a/t4_devkit/sanity/record/rec004.py +++ b/t4_devkit/sanity/record/rec004.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC004(RecordCountChecker): id = RuleID("REC004") name = RuleName("ego-pose-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'EgoPose' record is not empty." schema = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/record/rec006.py b/t4_devkit/sanity/record/rec006.py index 1ed9ecc9..09ce9183 100644 --- a/t4_devkit/sanity/record/rec006.py +++ b/t4_devkit/sanity/record/rec006.py @@ -6,7 +6,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -25,6 +25,7 @@ class REC006(RecordCountChecker): id = RuleID("REC006") name = RuleName("instance-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = ( "'Instance' record is not empty if either 'SampleAnnotation' or 'ObjectAnn' is not empty." ) @@ -45,8 +46,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case (None, _) | (_, None): return Maybe.from_value(Reason("Missing 'annotation' directory")) case (s, o): - sample_ann_count = len(load_json_safe(s).unwrap()) if s.exists() else 0 - object_ann_count = len(load_json_safe(o).unwrap()) if o.exists() else 0 + sample_ann_count = len(load_json_safe(s).value_or([])) if s.exists() else 0 + object_ann_count = len(load_json_safe(o).value_or([])) if o.exists() else 0 return ( Maybe.from_value( Reason( diff --git a/t4_devkit/sanity/record/rec007.py b/t4_devkit/sanity/record/rec007.py index 61a88d20..c9deb330 100644 --- a/t4_devkit/sanity/record/rec007.py +++ b/t4_devkit/sanity/record/rec007.py @@ -3,10 +3,11 @@ from typing import TYPE_CHECKING from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful from t4_devkit.schema import SchemaName -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -24,6 +25,7 @@ class REC007(Checker): id = RuleID("REC007") name = RuleName("category-indices-consistent") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "All categories must either have a unique 'index' or all have a 'null' index." schema = SchemaName.CATEGORY @@ -32,6 +34,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case Some(x): if not x.exists(): return Maybe.from_value(Reason(f"Missing {self.schema.filename}")) + elif not is_successful(load_json_safe(x)): + return Maybe.from_value(Reason(f"Cannot read {self.schema.filename}")) else: return Nothing case _: diff --git a/t4_devkit/sanity/record/rec008.py b/t4_devkit/sanity/record/rec008.py new file mode 100644 index 00000000..62c42a36 --- /dev/null +++ b/t4_devkit/sanity/record/rec008.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC008"] + + +@CHECKERS.register() +class REC008(RecordsChecker): + """A checker of REC008.""" + + id = RuleID("REC008") + name = RuleName("sensor-not-empty") + severity = Severity.ERROR + description = "'Sensor' record is not empty." + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + if not context.load_records(SchemaName.SENSOR).unwrap(): + return [Reason("'Sensor' record must not be empty")] + return None diff --git a/t4_devkit/sanity/record/rec009.py b/t4_devkit/sanity/record/rec009.py new file mode 100644 index 00000000..67beca3a --- /dev/null +++ b/t4_devkit/sanity/record/rec009.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import Counter + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC009"] + + +@CHECKERS.register() +class REC009(RecordsChecker): + """A checker of REC009.""" + + id = RuleID("REC009") + name = RuleName("sensor-index-unique") + severity = Severity.ERROR + description = "'Sensor.index' values are unique." + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + counts = Counter( + sensor.index for sensor in context.load_records(SchemaName.SENSOR).unwrap() + ) + return [ + Reason(f"'Sensor.index' {index} is used by {count} sensors") + for index, count in sorted(counts.items()) + if count > 1 + ] or None diff --git a/t4_devkit/sanity/record/rec010.py b/t4_devkit/sanity/record/rec010.py new file mode 100644 index 00000000..f997ab8d --- /dev/null +++ b/t4_devkit/sanity/record/rec010.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import Counter + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC010"] + + +@CHECKERS.register() +class REC010(RecordsChecker): + """A checker of REC010.""" + + id = RuleID("REC010") + name = RuleName("instance-index-unique") + severity = Severity.ERROR + description = "'Instance.index' values are unique where set." + tables = (SchemaName.INSTANCE,) + + def check(self, context: SanityContext) -> list[Reason] | None: + counts = Counter( + instance.index + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + if instance.index is not None + ) + return [ + Reason(f"'Instance.index' {index} is used by {count} instances") + for index, count in sorted(counts.items()) + if count > 1 + ] or None diff --git a/t4_devkit/sanity/record/rec011.py b/t4_devkit/sanity/record/rec011.py new file mode 100644 index 00000000..4b033a41 --- /dev/null +++ b/t4_devkit/sanity/record/rec011.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC011"] + + +@CHECKERS.register() +class REC011(RecordsChecker): + """A checker of REC011.""" + + id = RuleID("REC011") + name = RuleName("point-label-indices-presence") + severity = Severity.ERROR + description = ( + "When a LiDAR file has 'label' / 'instance' columns, every 'Category' has an 'index', " + "and every 'Instance' of an 'is_thing' category an 'index'." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.CATEGORY, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + columns = { + field.name + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + if sensor.lidar is not None + for field in sensor.lidar.fields + } + categories = context.load_records(SchemaName.CATEGORY).unwrap() + failures = [] + if "label" in columns: + failures += [ + Reason(f"Category without 'index': {category.name}") + for category in categories + if category.index is None + ] + if "instance" in columns: + things = {category.token for category in categories if category.is_thing} + failures += [ + Reason(f"Instance without 'index': {instance.token}") + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + if instance.category_token in things and instance.index is None + ] + return failures or None diff --git a/t4_devkit/sanity/record/rec012.py b/t4_devkit/sanity/record/rec012.py new file mode 100644 index 00000000..3cacd465 --- /dev/null +++ b/t4_devkit/sanity/record/rec012.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import Counter + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC012"] + + +@CHECKERS.register() +class REC012(RecordsChecker): + """A checker of REC012.""" + + id = RuleID("REC012") + name = RuleName("scene-sample-count") + severity = Severity.ERROR + description = "'Scene.nbr_samples' equals the number of 'Sample' records of the scene." + tables = ( + SchemaName.SCENE, + SchemaName.SAMPLE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + counts = self._sample_counts(context) + return [ + Reason( + f"Scene {scene.token}: 'nbr_samples' {scene.nbr_samples}, has {counts[scene.token]}" + ) + for scene in context.load_records(SchemaName.SCENE).unwrap() + if scene.nbr_samples != counts[scene.token] + ] or None + + def fix(self, context: SanityContext) -> bool: + counts = self._sample_counts(context) + records = context.load_table(SchemaName.SCENE).unwrap() + for record in records: + record["nbr_samples"] = counts[record["token"]] + return context.save_table(SchemaName.SCENE, records) + + @staticmethod + def _sample_counts(context: SanityContext) -> Counter[str]: + """Return the number of samples of each scene.""" + return Counter( + sample.scene_token for sample in context.load_records(SchemaName.SAMPLE).unwrap() + ) diff --git a/t4_devkit/sanity/record/rec013.py b/t4_devkit/sanity/record/rec013.py new file mode 100644 index 00000000..034b57b5 --- /dev/null +++ b/t4_devkit/sanity/record/rec013.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import Instance, SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC013"] + + +@CHECKERS.register() +class REC013(RecordsChecker): + """A checker of REC013.""" + + id = RuleID("REC013") + name = RuleName("instance-annotation-count") + severity = Severity.ERROR + description = ( + "'Instance.nbr_annotations' equals the number of its 'SampleAnnotation' and 'ObjectAnn' " + "records, and its first / last annotation tokens point to its first / last one in time." + ) + tables = ( + SchemaName.INSTANCE, + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + expected = self._expected(context) + failures = [] + for instance in context.load_records(SchemaName.INSTANCE).unwrap(): + count, first, last = expected.get(instance.token, (0, None, None)) + if instance.nbr_annotations != count: + failures.append( + Reason( + f"Instance {instance.token}: 'nbr_annotations' {instance.nbr_annotations}, has {count}" + ) + ) + if (instance.first_annotation_token, instance.last_annotation_token) != (first, last): + failures.append( + Reason( + f"Instance {instance.token}: first / last annotation " + f"{instance.first_annotation_token} / {instance.last_annotation_token}, " + f"expected {first} / {last}" + ) + ) + return failures or None + + def fix(self, context: SanityContext) -> bool: + expected = self._expected(context) + records = context.load_table(SchemaName.INSTANCE).unwrap() + for record in records: + count, first, last = expected.get(record["token"], (0, None, None)) + record["nbr_annotations"] = count + record["first_annotation_token"] = first + record["last_annotation_token"] = last + return context.save_table(SchemaName.INSTANCE, records) + + @staticmethod + def _expected(context: SanityContext) -> dict[str, tuple[int, str, str]]: + """Return the count and first / last label token of each instance with labels.""" + samples = {s.token: s for s in context.load_records(SchemaName.SAMPLE).unwrap()} + frames = {f.token: f for f in context.load_records(SchemaName.SAMPLE_DATA).unwrap()} + boxes = [ + (ann.instance_token, samples[ann.sample_token].timestamp, ann.token) + for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + if ann.sample_token in samples + ] + labels_2d = [ + (ann.instance_token, frames[ann.sample_data_token].timestamp, ann.token) + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap() + if ann.instance_token is not None and ann.sample_data_token in frames + ] + return Instance.annotation_spans(boxes + labels_2d) diff --git a/t4_devkit/sanity/record/rec014.py b/t4_devkit/sanity/record/rec014.py new file mode 100644 index 00000000..c831de74 --- /dev/null +++ b/t4_devkit/sanity/record/rec014.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC014"] + + +@CHECKERS.register() +class REC014(RecordsChecker): + """A checker of REC014.""" + + id = RuleID("REC014") + name = RuleName("thing-category-instance") + severity = Severity.ERROR + description = ( + "'ObjectAnn.instance_token' is set exactly when its category has 'is_thing'; " + "'SampleAnnotation' and 'Keypoint' instances belong to 'is_thing' categories." + ) + tables = ( + SchemaName.CATEGORY, + SchemaName.INSTANCE, + SchemaName.OBJECT_ANN, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.KEYPOINT, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + is_thing = { + category.token: category.is_thing + for category in context.load_records(SchemaName.CATEGORY).unwrap() + } + instance_category = { + instance.token: instance.category_token + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + } + failures = [] + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap(): + thing = is_thing.get(ann.category_token) + if thing is not None and thing != (ann.instance_token is not None): + expected = "an instance" if thing else "no instance" + failures.append(Reason(f"ObjectAnn {ann.token}: its category needs {expected}")) + for table in (SchemaName.SAMPLE_ANNOTATION, SchemaName.KEYPOINT): + for ann in context.load_records(table).unwrap(): + category = instance_category.get(ann.instance_token) + if category in is_thing and not is_thing[category]: + failures.append( + Reason( + f"{record_name(table)} {ann.token}: instance of a category without 'is_thing'" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec015.py b/t4_devkit/sanity/record/rec015.py new file mode 100644 index 00000000..53b58de2 --- /dev/null +++ b/t4_devkit/sanity/record/rec015.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC015"] + + +@CHECKERS.register() +class REC015(RecordsChecker): + """A checker of REC015.""" + + id = RuleID("REC015") + name = RuleName("orientation-number-allowed") + severity = Severity.ERROR + description = ( + "'ObjectAnn.orientation' / 'number' are set only for categories with " + "'has_orientation' / 'has_number'." + ) + tables = ( + SchemaName.CATEGORY, + SchemaName.OBJECT_ANN, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + categories = { + category.token: category + for category in context.load_records(SchemaName.CATEGORY).unwrap() + } + failures = [] + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap(): + category = categories.get(ann.category_token) + if category is None: + continue # reported by the reference rules + if ann.orientation is not None and not category.has_orientation: + failures.append(Reason(f"ObjectAnn {ann.token}: 'orientation' for {category.name}")) + if ann.number is not None and not category.has_number: + failures.append(Reason(f"ObjectAnn {ann.token}: 'number' for {category.name}")) + return failures or None diff --git a/t4_devkit/sanity/record/rec016.py b/t4_devkit/sanity/record/rec016.py new file mode 100644 index 00000000..df512b3d --- /dev/null +++ b/t4_devkit/sanity/record/rec016.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import AnnotationType, SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC016"] + + +@CHECKERS.register() +class REC016(RecordsChecker): + """A checker of REC016.""" + + id = RuleID("REC016") + name = RuleName("manual-label-confidence") + severity = Severity.ERROR + description = "Labels with 'annotation_type: manual' have 'confidence: null'." + tables = ( + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.KEYPOINT, + SchemaName.TRAFFIC_LIGHT, + SchemaName.SCENARIO, + SchemaName.CAPTION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return [ + Reason( + f"{record_name(table)} {label.token}: manual label with 'confidence' {label.confidence}" + ) + for table in self.tables + for label in context.load_records(table).unwrap() + if label.annotation_type == AnnotationType.MANUAL + and getattr(label, "confidence", None) is not None + ] or None diff --git a/t4_devkit/sanity/record/rec017.py b/t4_devkit/sanity/record/rec017.py new file mode 100644 index 00000000..6454cafb --- /dev/null +++ b/t4_devkit/sanity/record/rec017.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import AnnotationType, SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC017"] + + +@CHECKERS.register() +class REC017(RecordsChecker): + """A checker of REC017.""" + + id = RuleID("REC017") + name = RuleName("automatic-label-provenance") + severity = Severity.WARNING + description = ( + "Labels that are not 'manual' have a 'provenance', and 'auto' / 'pseudo' labels a " + "'confidence'." + ) + tables = ( + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.KEYPOINT, + SchemaName.TRAFFIC_LIGHT, + SchemaName.SCENARIO, + SchemaName.CAPTION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + for table in self.tables: + for label in context.load_records(table).unwrap(): + if label.annotation_type == AnnotationType.MANUAL: + continue + if label.provenance is None: + failures.append(Reason(f"{record_name(table)} {label.token}: no 'provenance'")) + scored = label.annotation_type in (AnnotationType.AUTO, AnnotationType.PSEUDO) + # `traffic_light` has no record-level confidence + if scored and hasattr(label, "confidence") and label.confidence is None: + failures.append(Reason(f"{record_name(table)} {label.token}: no 'confidence'")) + return failures or None diff --git a/t4_devkit/sanity/record/rec018.py b/t4_devkit/sanity/record/rec018.py new file mode 100644 index 00000000..9f790534 --- /dev/null +++ b/t4_devkit/sanity/record/rec018.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING + +from t4_devkit.schema import SensorModality +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC018"] + + +@CHECKERS.register() +class REC018(RecordsChecker): + """A checker of REC018.""" + + id = RuleID("REC018") + name = RuleName("sample-frames") + severity = Severity.ERROR + description = ( + "A 'Sample' has at most one 'SampleData' per channel; its 'timestamp' is its LiDAR frame's " + "(its earliest frame's if it has no LiDAR); 'has_key_frame' is true exactly when one of " + "its 'SampleData' is a key frame." + ) + tables = (SchemaName.SAMPLE, SchemaName.SAMPLE_DATA, SchemaName.SENSOR) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = { + sensor.token: sensor for sensor in context.load_records(SchemaName.SENSOR).unwrap() + } + frames: dict[str, list] = defaultdict(list) + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + if frame.sensor_token in sensors: # unknown sensors: reported by the reference rules + frames[frame.sample_token].append(frame) + + failures = [] + for sample in context.load_records(SchemaName.SAMPLE).unwrap(): + sample_frames = frames[sample.token] + channels = [sensors[frame.sensor_token].channel for frame in sample_frames] + for channel in sorted({c for c in channels if channels.count(c) > 1}): + failures.append(Reason(f"Sample {sample.token}: several frames of {channel}")) + if sample_frames: + lidar_frames = [ + frame + for frame in sample_frames + if sensors[frame.sensor_token].modality == SensorModality.LIDAR + ] + expected = min(frame.timestamp for frame in lidar_frames or sample_frames) + if sample.timestamp != expected: + failures.append( + Reason( + f"Sample {sample.token}: 'timestamp' {sample.timestamp}, expected {expected}" + ) + ) + has_key_frame = any(frame.is_key_frame for frame in sample_frames) + if sample.has_key_frame != has_key_frame: + failures.append( + Reason(f"Sample {sample.token}: 'has_key_frame' should be {has_key_frame}") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec019.py b/t4_devkit/sanity/record/rec019.py new file mode 100644 index 00000000..90e35df7 --- /dev/null +++ b/t4_devkit/sanity/record/rec019.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from .base import channel_file + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC019"] + + +@CHECKERS.register() +class REC019(RecordsChecker): + """A checker of REC019.""" + + id = RuleID("REC019") + name = RuleName("channel-frames") + severity = Severity.ERROR + description = ( + "For a channel stored in one file, 'frame_count' equals the number of its 'SampleData' " + "records, and their 'frame_index' values are 0 to 'frame_count - 1', each once." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = context.load_records(SchemaName.SENSOR).unwrap() + indices: dict[str, list[int | None]] = defaultdict(list) + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + indices[frame.sensor_token].append(frame.frame_index) + failures = [] + for sensor in sensors: + stored = channel_file(sensor) + # per-frame files, and source LiDARs of a merged channel (no file of their own) + if stored is None: + continue + _, frame_count = stored + if sorted(indices[sensor.token], key=lambda i: -1 if i is None else i) != list( + range(frame_count) + ): + failures.append( + Reason( + f"Sensor {sensor.channel} ({sensor.token}): 'frame_count' {frame_count}, " + f"{len(indices[sensor.token])} frames, 'frame_index' not 0 to {frame_count - 1}" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec020.py b/t4_devkit/sanity/record/rec020.py new file mode 100644 index 00000000..183f1de7 --- /dev/null +++ b/t4_devkit/sanity/record/rec020.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from .base import channel_file, frame_formats + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC020"] + + +@CHECKERS.register() +class REC020(RecordsChecker): + """A checker of REC020.""" + + id = RuleID("REC020") + name = RuleName("sensor-storage-consistent") + severity = Severity.ERROR + description = ( + "'SampleData.frame_index' is 'null' exactly when its sensor stores one file per frame, " + "and 'fileformat' matches the sensor's storage." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = { + sensor.token: sensor for sensor in context.load_records(SchemaName.SENSOR).unwrap() + } + failures = [] + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + sensor = sensors.get(frame.sensor_token) + if sensor is None: + continue # reported by the reference rules + one_file = channel_file(sensor) is not None + if one_file != (frame.frame_index is not None): + expected = "a 'frame_index'" if one_file else "no 'frame_index'" + failures.append(Reason(f"SampleData {frame.token}: needs {expected}")) + formats = frame_formats(sensor) + if frame.fileformat.value not in formats: + failures.append( + Reason( + f"SampleData {frame.token}: 'fileformat' {frame.fileformat.value}, " + f"sensor {sensor.channel} stores {' / '.join(sorted(formats))}" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec021.py b/t4_devkit/sanity/record/rec021.py new file mode 100644 index 00000000..0e88fcc6 --- /dev/null +++ b/t4_devkit/sanity/record/rec021.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC021"] + + +@CHECKERS.register() +class REC021(RecordsChecker): + """A checker of REC021.""" + + id = RuleID("REC021") + name = RuleName("timestamps-ordered") + severity = Severity.ERROR + description = ( + "Timestamps increase along 'next' ('Sample', 'SampleData', 'SampleAnnotation'), and " + "'frame_index' follows timestamp order within a channel file." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + samples = { + sample.token: sample for sample in context.load_records(SchemaName.SAMPLE).unwrap() + } + frames = { + frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + } + anns = { + ann.token: ann for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + } + + def ann_time(token: str) -> int | None: + sample = samples.get(anns[token].sample_token) + return None if sample is None else sample.timestamp + + failures = [] + for name, records, time in ( + ("Sample", samples, lambda token: samples[token].timestamp), + ("SampleData", frames, lambda token: frames[token].timestamp), + ("SampleAnnotation", anns, ann_time), + ): + for token, record in records.items(): + if record.next in records: + current, following = time(token), time(record.next) + if current is not None and following is not None and following <= current: + failures.append(Reason(f"{name} {token}: 'next' is not later")) + + by_sensor: dict[str, list] = defaultdict(list) + for frame in frames.values(): + if frame.frame_index is not None: + by_sensor[frame.sensor_token].append(frame) + for sensor_token, sensor_frames in by_sensor.items(): + ordered = sorted(sensor_frames, key=lambda frame: frame.frame_index) + if any(a.timestamp >= b.timestamp for a, b in zip(ordered, ordered[1:])): + failures.append( + Reason(f"Sensor {sensor_token}: 'frame_index' not in timestamp order") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec022.py b/t4_devkit/sanity/record/rec022.py new file mode 100644 index 00000000..073a1551 --- /dev/null +++ b/t4_devkit/sanity/record/rec022.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC022"] + + +@CHECKERS.register() +class REC022(RecordsChecker): + """A checker of REC022.""" + + id = RuleID("REC022") + name = RuleName("timestamps-microseconds") + severity = Severity.WARNING + description = ( + "Timestamps are Unix microseconds: between 2000-01-01 and 2100-01-01 (catches seconds or " + "nanoseconds)." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.EGO_POSE, + SchemaName.VEHICLE_STATE, + SchemaName.TRAFFIC_LIGHT, + SchemaName.ROUTE, + SchemaName.SCENARIO, + SchemaName.CAPTION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + # 2000-01-01 and 2100-01-01 in Unix microseconds + low, high = 946_684_800_000_000, 4_102_444_800_000_000 + fields_of = { + SchemaName.SAMPLE: ("timestamp",), + SchemaName.SAMPLE_DATA: ("timestamp",), + SchemaName.EGO_POSE: ("timestamp",), + SchemaName.VEHICLE_STATE: ("timestamp",), + SchemaName.TRAFFIC_LIGHT: ("timestamp",), + SchemaName.ROUTE: ("valid_from", "valid_until"), + SchemaName.SCENARIO: ("start_timestamp", "end_timestamp", "key_timestamp"), + SchemaName.CAPTION: ("start_timestamp", "end_timestamp"), + } + failures = [] + for table in self.tables: + for record in context.load_records(table).unwrap(): + for name in fields_of[table]: + value = getattr(record, name) + if value is not None and not low <= value < high: + failures.append( + Reason(f"{record_name(table)} {record.token}: '{name}' {value}") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec023.py b/t4_devkit/sanity/record/rec023.py new file mode 100644 index 00000000..061a91f5 --- /dev/null +++ b/t4_devkit/sanity/record/rec023.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from .base import capability_items, present_capabilities + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC023"] + + +@CHECKERS.register() +class REC023(RecordsChecker): + """A checker of REC023.""" + + id = RuleID("REC023") + name = RuleName("capabilities-present") + severity = Severity.ERROR + description = ( + "Everything 'manifest.json' 'capabilities' declares is present: channels, map, tables and " + "the labels of its tasks." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.MAP, + SchemaName.ROUTE, + SchemaName.TRAFFIC_LIGHT, + SchemaName.VEHICLE_STATE, + SchemaName.SCENARIO, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.CATEGORY, + SchemaName.KEYPOINT, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + present = present_capabilities(context, self.tables) + return [ + Reason(f"Declared but not present: {item}") + for item in sorted(capability_items(context.manifest.unwrap().capabilities) - present) + ] or None diff --git a/t4_devkit/sanity/record/rec024.py b/t4_devkit/sanity/record/rec024.py new file mode 100644 index 00000000..54791d0f --- /dev/null +++ b/t4_devkit/sanity/record/rec024.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from .base import capability_items, present_capabilities + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC024"] + + +@CHECKERS.register() +class REC024(RecordsChecker): + """A checker of REC024.""" + + id = RuleID("REC024") + name = RuleName("capabilities-complete") + severity = Severity.WARNING + description = ( + "Everything present is declared in 'capabilities': the same checks as REC023, the other " + "way round." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.MAP, + SchemaName.ROUTE, + SchemaName.TRAFFIC_LIGHT, + SchemaName.VEHICLE_STATE, + SchemaName.SCENARIO, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.CATEGORY, + SchemaName.KEYPOINT, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + declared = capability_items(context.manifest.unwrap().capabilities) + return [ + Reason(f"Present but not declared: {item}") + for item in sorted(present_capabilities(context, self.tables) - declared) + ] or None diff --git a/t4_devkit/sanity/record/rec025.py b/t4_devkit/sanity/record/rec025.py new file mode 100644 index 00000000..93967873 --- /dev/null +++ b/t4_devkit/sanity/record/rec025.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName, V0Converter + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC025"] + + +@CHECKERS.register() +class REC025(RecordsChecker): + """A checker of REC025.""" + + id = RuleID("REC025") + name = RuleName("no-placeholder-values") + severity = Severity.WARNING + description = ( + "No placeholder values from loading or migrating 0.x: 'vehicle.model' and 'map.map_id' / " + "'version_id' are not 'unknown'." + ) + tables = ( + SchemaName.VEHICLE, + SchemaName.MAP, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + vehicle_model = V0Converter.FALLBACKS["vehicle"]["model"] + map_fallbacks = V0Converter.FALLBACKS["map"] + failures = [ + Reason(f"Vehicle {vehicle.token}: placeholder 'model' {vehicle.model!r}") + for vehicle in context.load_records(SchemaName.VEHICLE).unwrap() + if vehicle.model == vehicle_model + ] + for map_record in context.load_records(SchemaName.MAP).unwrap(): + for name in ("map_id", "version_id"): + value = getattr(map_record, name) + if value == map_fallbacks[name]: + failures.append( + Reason(f"Map {map_record.token}: placeholder '{name}' {value!r}") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec026.py b/t4_devkit/sanity/record/rec026.py new file mode 100644 index 00000000..b886bb15 --- /dev/null +++ b/t4_devkit/sanity/record/rec026.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC026"] + + +@CHECKERS.register() +class REC026(RecordsChecker): + """A checker of REC026.""" + + id = RuleID("REC026") + name = RuleName("route-validity") + severity = Severity.ERROR + description = ( + "'route.valid_until' is after 'valid_from' when set, and the routes of a log do not overlap " + "in time." + ) + tables = (SchemaName.ROUTE,) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + by_log: dict[str, list] = defaultdict(list) + for route in context.load_records(SchemaName.ROUTE).unwrap(): + if route.valid_until is not None and route.valid_until <= route.valid_from: + failures.append( + Reason(f"Route {route.token}: 'valid_until' not after 'valid_from'") + ) + by_log[route.log_token].append(route) + for routes in by_log.values(): + ordered = sorted(routes, key=lambda route: route.valid_from) + for route, following in zip(ordered, ordered[1:]): + if route.valid_until is None or route.valid_until > following.valid_from: + failures.append(Reason(f"Routes {route.token} and {following.token} overlap")) + return failures or None diff --git a/t4_devkit/sanity/record/rec027.py b/t4_devkit/sanity/record/rec027.py new file mode 100644 index 00000000..172cb7ab --- /dev/null +++ b/t4_devkit/sanity/record/rec027.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC027"] + + +@CHECKERS.register() +class REC027(RecordsChecker): + """A checker of REC027.""" + + id = RuleID("REC027") + name = RuleName("unit-quaternions") + severity = Severity.WARNING + description = ( + "Rotations ('Sensor', 'EgoPose', 'SampleAnnotation', 'Route' poses) are unit quaternions." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.EGO_POSE, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.ROUTE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + rotations = [ + (record_name(table), record.token, record.rotation) + for table in (SchemaName.SENSOR, SchemaName.EGO_POSE, SchemaName.SAMPLE_ANNOTATION) + for record in context.load_records(table).unwrap() + ] + [ + (record_name(SchemaName.ROUTE), route.token, pose.rotation) + for route in context.load_records(SchemaName.ROUTE).unwrap() + for pose in (route.start_pose, route.goal_pose) + ] + tolerance = 1e-3 + return [ + Reason(f"{table} {token}: rotation norm {rotation.norm:.6f}") + for table, token, rotation in rotations + if abs(rotation.norm - 1.0) > tolerance + ] or None diff --git a/t4_devkit/sanity/record/rec028.py b/t4_devkit/sanity/record/rec028.py new file mode 100644 index 00000000..2e53bdfd --- /dev/null +++ b/t4_devkit/sanity/record/rec028.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC028"] + + +@CHECKERS.register() +class REC028(RecordsChecker): + """A checker of REC028.""" + + id = RuleID("REC028") + name = RuleName("labels-on-key-frames") + severity = Severity.ERROR + description = ( + "Labels sit on key frames: a 'SampleAnnotation' on a 'Sample' with 'has_key_frame', an " + "'ObjectAnn' or 'Keypoint' on a 'SampleData' with 'is_key_frame'." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.KEYPOINT, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + key_samples = { + sample.token + for sample in context.load_records(SchemaName.SAMPLE).unwrap() + if sample.has_key_frame + } + key_frames = { + frame.token + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + if frame.is_key_frame + } + failures = [ + Reason(f"SampleAnnotation {ann.token}: sample without a key frame") + for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + if ann.sample_token not in key_samples + ] + for table in (SchemaName.OBJECT_ANN, SchemaName.KEYPOINT): + failures += [ + Reason(f"{record_name(table)} {ann.token}: not on a key frame") + for ann in context.load_records(table).unwrap() + if ann.sample_data_token not in key_frames + ] + return failures or None diff --git a/t4_devkit/sanity/record/rec029.py b/t4_devkit/sanity/record/rec029.py new file mode 100644 index 00000000..c313fd89 --- /dev/null +++ b/t4_devkit/sanity/record/rec029.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..reference.files import lidar_partly_null_frames +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC029"] + + +@CHECKERS.register() +class REC029(RecordsChecker): + """A checker of REC029.""" + + id = RuleID("REC029") + name = RuleName("point-labels-reserved") + severity = Severity.ERROR + description = ( + "When a LiDAR file has 'label' / 'instance' columns: the 'Category' with 'index' 0 is " + "named 'unpainted', and each frame's label columns are null on all points or on none." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.CATEGORY, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = [s for s in context.load_records(SchemaName.SENSOR).unwrap() if s.lidar] + columns = {field.name for sensor in sensors for field in sensor.lidar.fields} + failures = [] + if "label" in columns: + names = [ + c.name for c in context.load_records(SchemaName.CATEGORY).unwrap() if c.index == 0 + ] + if not names: + failures.append(Reason("No 'Category' with 'index' 0 ('unpainted')")) + elif names != ["unpainted"]: + failures.append(Reason(f"'Category' with 'index' 0 is {names}, not 'unpainted'")) + + data_root = context.data_root.unwrap() + for sensor in sensors: + if sensor.lidar.container.path is None: + continue + path = data_root.joinpath(sensor.lidar.container.path) + if not path.exists(): + continue + for column in ("label", "instance"): + frames = lidar_partly_null_frames(path, column) + if frames: + failures.append( + Reason( + f"Sensor {sensor.channel}: '{column}' is null on only some points of " + f"frames {frames[:10]}" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/reference/__init__.py b/t4_devkit/sanity/reference/__init__.py index b9fbaca5..74adcd09 100644 --- a/t4_devkit/sanity/reference/__init__.py +++ b/t4_devkit/sanity/reference/__init__.py @@ -24,3 +24,25 @@ from .ref202 import * # noqa from .ref203 import * # noqa from .ref301 import * # noqa +from .ref015 import * # noqa +from .ref016 import * # noqa +from .ref017 import * # noqa +from .ref018 import * # noqa +from .ref019 import * # noqa +from .ref020 import * # noqa +from .ref021 import * # noqa +from .ref022 import * # noqa +from .ref023 import * # noqa +from .ref024 import * # noqa +from .ref025 import * # noqa +from .ref107 import * # noqa +from .ref108 import * # noqa +from .ref204 import * # noqa +from .ref205 import * # noqa +from .ref206 import * # noqa +from .ref207 import * # noqa +from .ref302 import * # noqa +from .ref303 import * # noqa +from .ref304 import * # noqa +from .ref305 import * # noqa +from .ref306 import * # noqa diff --git a/t4_devkit/sanity/reference/base.py b/t4_devkit/sanity/reference/base.py index 5b32f76e..961b7c14 100644 --- a/t4_devkit/sanity/reference/base.py +++ b/t4_devkit/sanity/reference/base.py @@ -5,12 +5,13 @@ from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful -from ..checker import Checker +from ..checker import Checker, record_name from ..result import Reason from ..safety import load_json_safe if TYPE_CHECKING: from t4_devkit.schema import SchemaName + from t4_devkit.schema.v1 import SchemaName as V1SchemaName from ..context import SanityContext @@ -41,6 +42,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: target_file_paths = [y.unwrap() for y in ys if is_successful(y)] if not x.exists(): return Maybe.from_value(Reason(f"Missing {self.source.filename}")) + elif not is_successful(load_json_safe(x)): + return Maybe.from_value(Reason(f"Cannot read {self.source.filename}")) elif not any(path.exists() for path in target_file_paths): return Maybe.from_value( Reason(f"Missing {', '.join(t.filename for t in target_schemas)}") @@ -59,18 +62,19 @@ def check(self, context: SanityContext) -> list[Reason] | None: if is_successful(target_file) and target_file.unwrap().exists() ] source_records = load_json_safe(source_file).unwrap() - target_tokens = [ + target_tokens = { item["token"] for target_file in target_files - for item in load_json_safe(target_file).unwrap() - ] + for item in load_json_safe(target_file).value_or([]) + } return [ Reason( f"No reference to '{self.source.value}.{self.reference}': {record[self.reference]}" ) for record in source_records if record[self.reference] not in target_tokens - and record[self.reference] != "" # NOTE: success if the reference token is "" + # NOTE: success if the reference token is "" (0.x) or null (1.0) + and record[self.reference] not in ("", None) and self.is_additional_condition_ok(record) ] or None @@ -120,3 +124,39 @@ class ExternalReferenceChecker(Checker): target: SchemaName reference: str + + +def missing_references( + context: SanityContext, + source: V1SchemaName, + fields: dict[str, V1SchemaName], +) -> list[Reason]: + """Return the 1.0 links of a table that refer to no record: one reason per missing token. + + Args: + context (SanityContext): The sanity context. + source (V1SchemaName): The table whose records hold the links. + fields (dict[str, V1SchemaName]): Link field (a token, a list of tokens, or null for no + link) and the table it refers to. + + Returns: + A reason per link to a missing record. + """ + tokens = { + table: {record.token for record in context.load_records(table).unwrap()} + for table in set(fields.values()) + } + reasons = [] + for record in context.load_records(source).unwrap(): + for name, table in fields.items(): + value = getattr(record, name) + values = value if isinstance(value, list) else [value] + reasons += [ + Reason( + f"{record_name(source)} {record.token}: '{name}' {token} refers to no " + f"{record_name(table)}" + ) + for token in values + if token is not None and token not in tokens[table] + ] + return reasons diff --git a/t4_devkit/sanity/reference/files.py b/t4_devkit/sanity/reference/files.py new file mode 100644 index 00000000..d0c97c95 --- /dev/null +++ b/t4_devkit/sanity/reference/files.py @@ -0,0 +1,105 @@ +"""Reading 1.0 channel files for the file reference rules.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import av +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from ..result import Reason + +if TYPE_CHECKING: + from pathlib import Path + + from t4_devkit.schema.v1 import Sensor + +__all__ = [ + "describe_lidar_file", + "describe_video_file", + "lidar_column_values", + "lidar_partly_null_frames", +] + + +def describe_lidar_file(sensor: Sensor, path: Path) -> list[Reason]: + """Return the differences between a LiDAR Parquet file and its sensor row (columns, dtypes, + frames).""" + parquet = pq.ParquetFile(path) + columns = [(field.name, _numpy_dtype(field.type)) for field in parquet.schema_arrow] + declared = [(field.name, field.dtype) for field in sensor.lidar.fields] + failures = [] + if columns != declared: + failures.append(Reason(f"Sensor {sensor.channel}: columns {columns}, declared {declared}")) + if parquet.num_row_groups != sensor.lidar.container.frame_count: + failures.append( + Reason( + f"Sensor {sensor.channel}: {parquet.num_row_groups} row groups, " + f"'frame_count' {sensor.lidar.container.frame_count}" + ) + ) + return failures + + +def describe_video_file(sensor: Sensor, path: Path) -> list[Reason]: + """Return the differences between a camera video and its sensor row (codec, pixels, timing, + size).""" + camera, video = sensor.camera, sensor.camera.video + with av.open(str(path)) as container: + stream = container.streams.video[0] + frame = next(container.decode(stream)) + actual = { + "codec": stream.codec_context.codec.name, + "pixel_format": frame.format.name, + "bit_depth": frame.format.components[0].bits, + "timescale": stream.time_base.denominator, + "frame_count": stream.frames, + "size": (frame.width, frame.height), + } + expected = { + "codec": "av1", + "pixel_format": video.pixel_format.split()[0], + "bit_depth": video.bit_depth, + "timescale": video.timescale, + "frame_count": video.frame_count, + "size": (camera.width, camera.height), + } + # decoders of AV1 have their own names (libdav1d, libaom-av1, ...) + if "av1" in actual["codec"]: + actual["codec"] = "av1" + return [ + Reason(f"Sensor {sensor.channel}: {name} {actual[name]}, declared {expected[name]}") + for name in expected + if actual[name] != expected[name] + ] + + +def lidar_column_values(path: Path, column: str) -> set | None: + """Return the distinct non-null values of a column of a LiDAR Parquet file, or None if it has + no such column.""" + parquet = pq.ParquetFile(path) + if column not in parquet.schema_arrow.names: + return None + values = pc.unique(parquet.read(columns=[column]).column(column)).drop_null() + return set(values.to_pylist()) + + +def lidar_partly_null_frames(path: Path, column: str) -> list[int] | None: + """Return the frames of a LiDAR Parquet file where a column is null on some points but not + all, or None if it has no such column.""" + parquet = pq.ParquetFile(path) + if column not in parquet.schema_arrow.names: + return None + frames = [] + for i in range(parquet.num_row_groups): + nulls = parquet.read_row_group(i, columns=[column]).column(column).null_count + if 0 < nulls < parquet.metadata.row_group(i).num_rows: + frames.append(i) + return frames + + +def _numpy_dtype(arrow_type: pa.DataType) -> str: + """Return the NumPy name of an Arrow type, as `sensor.lidar.fields` writes it (e.g. "float32").""" + return pa.array([], type=arrow_type).to_numpy(zero_copy_only=False).dtype.name diff --git a/t4_devkit/sanity/reference/ref001.py b/t4_devkit/sanity/reference/ref001.py index c4f69e95..94f63422 100644 --- a/t4_devkit/sanity/reference/ref001.py +++ b/t4_devkit/sanity/reference/ref001.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF001(RecordReferenceChecker): id = RuleID("REF001") name = RuleName("scene-to-log") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene.log_token' refers to 'Log' record." source = SchemaName.SCENE target = SchemaName.LOG diff --git a/t4_devkit/sanity/reference/ref002.py b/t4_devkit/sanity/reference/ref002.py index 0ad3a835..1a38ee2c 100644 --- a/t4_devkit/sanity/reference/ref002.py +++ b/t4_devkit/sanity/reference/ref002.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF002(RecordReferenceChecker): id = RuleID("REF002") name = RuleName("scene-to-first-sample") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene.first_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref003.py b/t4_devkit/sanity/reference/ref003.py index 420d231c..1414118f 100644 --- a/t4_devkit/sanity/reference/ref003.py +++ b/t4_devkit/sanity/reference/ref003.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF003(RecordReferenceChecker): id = RuleID("REF003") name = RuleName("scene-to-last-sample") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene.last_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref004.py b/t4_devkit/sanity/reference/ref004.py index a8bea29a..533b1c65 100644 --- a/t4_devkit/sanity/reference/ref004.py +++ b/t4_devkit/sanity/reference/ref004.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF004(RecordReferenceChecker): id = RuleID("REF004") name = RuleName("sample-to-scene") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample.scene_token' refers to 'Scene' record." source = SchemaName.SAMPLE target = SchemaName.SCENE diff --git a/t4_devkit/sanity/reference/ref005.py b/t4_devkit/sanity/reference/ref005.py index 13ca061e..358d77c7 100644 --- a/t4_devkit/sanity/reference/ref005.py +++ b/t4_devkit/sanity/reference/ref005.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -18,6 +18,7 @@ class REF005(RecordReferenceChecker): id = RuleID("REF005") name = RuleName("sample-data-to-sample") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.sample_token' refers to 'Sample' record." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref006.py b/t4_devkit/sanity/reference/ref006.py index 01dbf62d..d44b926e 100644 --- a/t4_devkit/sanity/reference/ref006.py +++ b/t4_devkit/sanity/reference/ref006.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF006(RecordReferenceChecker): id = RuleID("REF006") name = RuleName("sample-data-to-ego-pose") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.ego_pose_token' refers to 'EgoPose' record." source = SchemaName.SAMPLE_DATA target = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/reference/ref009.py b/t4_devkit/sanity/reference/ref009.py index 61257e55..2ce220e2 100644 --- a/t4_devkit/sanity/reference/ref009.py +++ b/t4_devkit/sanity/reference/ref009.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF009(RecordReferenceChecker): id = RuleID("REF009") name = RuleName("instance-to-category") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Instance.category_token' refers to 'Category' record." source = SchemaName.INSTANCE target = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/reference/ref010.py b/t4_devkit/sanity/reference/ref010.py index daf38526..01eed0a9 100644 --- a/t4_devkit/sanity/reference/ref010.py +++ b/t4_devkit/sanity/reference/ref010.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF010(RecordReferenceChecker): id = RuleID("REF010") name = RuleName("instance-to-first-sample-annotation") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = ( "'Instance.first_annotation_token' refers to 'SampleAnnotation' or 'ObjectAnn' record." ) diff --git a/t4_devkit/sanity/reference/ref011.py b/t4_devkit/sanity/reference/ref011.py index 36f04956..f1a83d8a 100644 --- a/t4_devkit/sanity/reference/ref011.py +++ b/t4_devkit/sanity/reference/ref011.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF011(RecordReferenceChecker): id = RuleID("REF011") name = RuleName("instance-to-last-sample-annotation") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = ( "'Instance.last_annotation_token' refers to 'SampleAnnotation' or 'ObjectAnn' record." ) diff --git a/t4_devkit/sanity/reference/ref015.py b/t4_devkit/sanity/reference/ref015.py new file mode 100644 index 00000000..aaf080a4 --- /dev/null +++ b/t4_devkit/sanity/reference/ref015.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF015"] + + +@CHECKERS.register() +class REF015(RecordsChecker): + """A checker of REF015.""" + + id = RuleID("REF015") + name = RuleName("sample-data-to-sensor") + severity = Severity.ERROR + description = "'SampleData.sensor_token' refers to 'Sensor' record." + tables = ( + SchemaName.SAMPLE_DATA, + SchemaName.SENSOR, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references(context, SchemaName.SAMPLE_DATA, {"sensor_token": SchemaName.SENSOR}) + or None + ) diff --git a/t4_devkit/sanity/reference/ref016.py b/t4_devkit/sanity/reference/ref016.py new file mode 100644 index 00000000..63081728 --- /dev/null +++ b/t4_devkit/sanity/reference/ref016.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF016"] + + +@CHECKERS.register() +class REF016(RecordsChecker): + """A checker of REF016.""" + + id = RuleID("REF016") + name = RuleName("log-to-vehicle") + severity = Severity.ERROR + description = "'Log.vehicle_token' refers to 'Vehicle' record." + tables = ( + SchemaName.LOG, + SchemaName.VEHICLE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references(context, SchemaName.LOG, {"vehicle_token": SchemaName.VEHICLE}) + or None + ) diff --git a/t4_devkit/sanity/reference/ref017.py b/t4_devkit/sanity/reference/ref017.py new file mode 100644 index 00000000..8e09d13f --- /dev/null +++ b/t4_devkit/sanity/reference/ref017.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF017"] + + +@CHECKERS.register() +class REF017(RecordsChecker): + """A checker of REF017.""" + + id = RuleID("REF017") + name = RuleName("log-and-map") + severity = Severity.ERROR + description = ( + "'Log.map_token' refers to 'Map' record unless it is empty, 'Map.log_tokens' refer to " + "'Log' records, and both agree." + ) + tables = ( + SchemaName.LOG, + SchemaName.MAP, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = missing_references(context, SchemaName.LOG, {"map_token": SchemaName.MAP}) + failures += missing_references(context, SchemaName.MAP, {"log_tokens": SchemaName.LOG}) + maps = {map_.token: map_ for map_ in context.load_records(SchemaName.MAP).unwrap()} + for log in context.load_records(SchemaName.LOG).unwrap(): + if log.map_token in maps and log.token not in maps[log.map_token].log_tokens: + failures.append(Reason(f"Log {log.token}: its map does not list it")) + for map_ in maps.values(): + if log.token in map_.log_tokens and log.map_token != map_.token: + failures.append(Reason(f"Log {log.token}: listed by map {map_.token}")) + return failures or None diff --git a/t4_devkit/sanity/reference/ref018.py b/t4_devkit/sanity/reference/ref018.py new file mode 100644 index 00000000..2e49d35c --- /dev/null +++ b/t4_devkit/sanity/reference/ref018.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF018"] + + +@CHECKERS.register() +class REF018(RecordsChecker): + """A checker of REF018.""" + + id = RuleID("REF018") + name = RuleName("sample-annotation-references") + severity = Severity.ERROR + description = ( + "'SampleAnnotation' 'sample_token', 'instance_token', 'attribute_tokens' and " + "'visibility_token' (unless empty) refer to records." + ) + tables = ( + SchemaName.ATTRIBUTE, + SchemaName.INSTANCE, + SchemaName.SAMPLE, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.VISIBILITY, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.SAMPLE_ANNOTATION, + { + "sample_token": SchemaName.SAMPLE, + "instance_token": SchemaName.INSTANCE, + "attribute_tokens": SchemaName.ATTRIBUTE, + "visibility_token": SchemaName.VISIBILITY, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref019.py b/t4_devkit/sanity/reference/ref019.py new file mode 100644 index 00000000..4d0c095b --- /dev/null +++ b/t4_devkit/sanity/reference/ref019.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF019"] + + +@CHECKERS.register() +class REF019(RecordsChecker): + """A checker of REF019.""" + + id = RuleID("REF019") + name = RuleName("object-ann-references") + severity = Severity.ERROR + description = ( + "'ObjectAnn' 'sample_data_token', 'instance_token' (unless empty), 'category_token' and " + "'attribute_tokens' refer to records." + ) + tables = ( + SchemaName.ATTRIBUTE, + SchemaName.CATEGORY, + SchemaName.INSTANCE, + SchemaName.OBJECT_ANN, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.OBJECT_ANN, + { + "sample_data_token": SchemaName.SAMPLE_DATA, + "instance_token": SchemaName.INSTANCE, + "category_token": SchemaName.CATEGORY, + "attribute_tokens": SchemaName.ATTRIBUTE, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref020.py b/t4_devkit/sanity/reference/ref020.py new file mode 100644 index 00000000..0f241cd7 --- /dev/null +++ b/t4_devkit/sanity/reference/ref020.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF020"] + + +@CHECKERS.register() +class REF020(RecordsChecker): + """A checker of REF020.""" + + id = RuleID("REF020") + name = RuleName("keypoint-references") + severity = Severity.ERROR + description = ( + "'Keypoint' 'sample_data_token', 'instance_token' and 'category_tokens' refer to records." + ) + tables = ( + SchemaName.CATEGORY, + SchemaName.INSTANCE, + SchemaName.KEYPOINT, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.KEYPOINT, + { + "sample_data_token": SchemaName.SAMPLE_DATA, + "instance_token": SchemaName.INSTANCE, + "category_tokens": SchemaName.CATEGORY, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref021.py b/t4_devkit/sanity/reference/ref021.py new file mode 100644 index 00000000..41ce35a4 --- /dev/null +++ b/t4_devkit/sanity/reference/ref021.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF021"] + + +@CHECKERS.register() +class REF021(RecordsChecker): + """A checker of REF021.""" + + id = RuleID("REF021") + name = RuleName("traffic-light-references") + severity = Severity.ERROR + description = ( + "'TrafficLight' 'object_ann_token' and 'instance_token' refer to records unless they are " + "empty." + ) + tables = ( + SchemaName.INSTANCE, + SchemaName.OBJECT_ANN, + SchemaName.TRAFFIC_LIGHT, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.TRAFFIC_LIGHT, + {"object_ann_token": SchemaName.OBJECT_ANN, "instance_token": SchemaName.INSTANCE}, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref022.py b/t4_devkit/sanity/reference/ref022.py new file mode 100644 index 00000000..b0609428 --- /dev/null +++ b/t4_devkit/sanity/reference/ref022.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF022"] + + +@CHECKERS.register() +class REF022(RecordsChecker): + """A checker of REF022.""" + + id = RuleID("REF022") + name = RuleName("route-references") + severity = Severity.ERROR + description = "'Route' 'log_token' and 'map_token' refer to records." + tables = ( + SchemaName.LOG, + SchemaName.MAP, + SchemaName.ROUTE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.ROUTE, + {"log_token": SchemaName.LOG, "map_token": SchemaName.MAP}, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref023.py b/t4_devkit/sanity/reference/ref023.py new file mode 100644 index 00000000..bc19af6c --- /dev/null +++ b/t4_devkit/sanity/reference/ref023.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF023"] + + +@CHECKERS.register() +class REF023(RecordsChecker): + """A checker of REF023.""" + + id = RuleID("REF023") + name = RuleName("scenario-references") + severity = Severity.ERROR + description = "'Scenario.log_token' refers to 'Log' record." + tables = ( + SchemaName.LOG, + SchemaName.SCENARIO, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references(context, SchemaName.SCENARIO, {"log_token": SchemaName.LOG}) or None + ) diff --git a/t4_devkit/sanity/reference/ref024.py b/t4_devkit/sanity/reference/ref024.py new file mode 100644 index 00000000..b4acddec --- /dev/null +++ b/t4_devkit/sanity/reference/ref024.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF024"] + + +@CHECKERS.register() +class REF024(RecordsChecker): + """A checker of REF024.""" + + id = RuleID("REF024") + name = RuleName("caption-references") + severity = Severity.ERROR + description = ( + "'Caption' 'log_token', 'sample_data_token', 'scenario_token' and 'instance_tokens' refer " + "to records unless they are empty." + ) + tables = ( + SchemaName.CAPTION, + SchemaName.INSTANCE, + SchemaName.LOG, + SchemaName.SAMPLE_DATA, + SchemaName.SCENARIO, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.CAPTION, + { + "log_token": SchemaName.LOG, + "sample_data_token": SchemaName.SAMPLE_DATA, + "scenario_token": SchemaName.SCENARIO, + "instance_tokens": SchemaName.INSTANCE, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref025.py b/t4_devkit/sanity/reference/ref025.py new file mode 100644 index 00000000..50f2dbfb --- /dev/null +++ b/t4_devkit/sanity/reference/ref025.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema import SensorModality +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF025"] + + +@CHECKERS.register() +class REF025(RecordsChecker): + """A checker of REF025.""" + + id = RuleID("REF025") + name = RuleName("lidar-source-channels") + severity = Severity.ERROR + description = "'Lidar.source_channels' name LiDAR 'Sensor' records that have no 'SampleData'." + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = context.load_records(SchemaName.SENSOR).unwrap() + with_frames = { + frame.sensor_token for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + } + sources = { + sensor.channel + for sensor in sensors + if sensor.modality == SensorModality.LIDAR and sensor.token not in with_frames + } + return [ + Reason( + f"Sensor {sensor.channel}: source channel {channel} is not a LiDAR without frames" + ) + for sensor in sensors + if sensor.lidar is not None + for channel in sensor.lidar.source_channels + if channel not in sources + ] or None diff --git a/t4_devkit/sanity/reference/ref101.py b/t4_devkit/sanity/reference/ref101.py index 4344f3c2..efe6bcc1 100644 --- a/t4_devkit/sanity/reference/ref101.py +++ b/t4_devkit/sanity/reference/ref101.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF101(RecordReferenceChecker): id = RuleID("REF101") name = RuleName("sample-next-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample.next' refers to another one unless it is empty." source = SchemaName.SAMPLE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref102.py b/t4_devkit/sanity/reference/ref102.py index abfaed49..7d90fb3e 100644 --- a/t4_devkit/sanity/reference/ref102.py +++ b/t4_devkit/sanity/reference/ref102.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF102(RecordReferenceChecker): id = RuleID("REF102") name = RuleName("sample-prev-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample.prev' refers to another one unless it is empty." source = SchemaName.SAMPLE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref103.py b/t4_devkit/sanity/reference/ref103.py index ee956b79..c59fc5df 100644 --- a/t4_devkit/sanity/reference/ref103.py +++ b/t4_devkit/sanity/reference/ref103.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF103(RecordReferenceChecker): id = RuleID("REF103") name = RuleName("sample-annotation-next-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleAnnotation.next' refers to another one unless it is empty." source = SchemaName.SAMPLE_ANNOTATION target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref104.py b/t4_devkit/sanity/reference/ref104.py index ea363587..f980e06e 100644 --- a/t4_devkit/sanity/reference/ref104.py +++ b/t4_devkit/sanity/reference/ref104.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF104(RecordReferenceChecker): id = RuleID("REF104") name = RuleName("sample-annotation-prev-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleAnnotation.prev' refers to another one unless it is empty." source = SchemaName.SAMPLE_ANNOTATION target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref105.py b/t4_devkit/sanity/reference/ref105.py index 172dc5ce..cd9f0034 100644 --- a/t4_devkit/sanity/reference/ref105.py +++ b/t4_devkit/sanity/reference/ref105.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF105(RecordReferenceChecker): id = RuleID("REF105") name = RuleName("sample-data-next-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.next' refers to another one unless it is empty." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref106.py b/t4_devkit/sanity/reference/ref106.py index c6f0c454..a543b720 100644 --- a/t4_devkit/sanity/reference/ref106.py +++ b/t4_devkit/sanity/reference/ref106.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF106(RecordReferenceChecker): id = RuleID("REF106") name = RuleName("sample-data-prev-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.prev' refers to another one unless it is empty." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref107.py b/t4_devkit/sanity/reference/ref107.py new file mode 100644 index 00000000..84de0aa1 --- /dev/null +++ b/t4_devkit/sanity/reference/ref107.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF107"] + + +@CHECKERS.register() +class REF107(RecordsChecker): + """A checker of REF107.""" + + id = RuleID("REF107") + name = RuleName("links-mutual") + severity = Severity.ERROR + description = ( + "'next' and 'prev' agree: 'A.next' has 'prev' = 'A' ('Sample', 'SampleData', " + "'SampleAnnotation')." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + for table in self.tables: + records = {record.token: record for record in context.load_records(table).unwrap()} + for record in records.values(): + following = records.get(record.next) + if following is not None and following.prev != record.token: + failures.append( + Reason(f"{record_name(table)} {record.token}: 'next' has another 'prev'") + ) + previous = records.get(record.prev) + if previous is not None and previous.next != record.token: + failures.append( + Reason(f"{record_name(table)} {record.token}: 'prev' has another 'next'") + ) + return failures or None diff --git a/t4_devkit/sanity/reference/ref108.py b/t4_devkit/sanity/reference/ref108.py new file mode 100644 index 00000000..6ae55d75 --- /dev/null +++ b/t4_devkit/sanity/reference/ref108.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF108"] + + +@CHECKERS.register() +class REF108(RecordsChecker): + """A checker of REF108.""" + + id = RuleID("REF108") + name = RuleName("links-same-stream") + severity = Severity.ERROR + description = ( + "'SampleData.next' is of the same channel, and 'SampleAnnotation.next' of the same " + "instance." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + channels = { + sensor.token: sensor.channel + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + } + frames = { + frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + } + anns = { + ann.token: ann for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + } + failures = [ + Reason(f"SampleData {frame.token}: 'next' is of another channel") + for frame in frames.values() + if frame.next in frames + and channels.get(frames[frame.next].sensor_token) != channels.get(frame.sensor_token) + ] + failures += [ + Reason(f"SampleAnnotation {ann.token}: 'next' is of another instance") + for ann in anns.values() + if ann.next in anns and anns[ann.next].instance_token != ann.instance_token + ] + return failures or None diff --git a/t4_devkit/sanity/reference/ref201.py b/t4_devkit/sanity/reference/ref201.py index 168ebd86..ca1b682d 100644 --- a/t4_devkit/sanity/reference/ref201.py +++ b/t4_devkit/sanity/reference/ref201.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -24,6 +24,7 @@ class REF201(FileReferenceChecker): id = RuleID("REF201") name = RuleName("sample-data-filename-presence") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA @@ -34,5 +35,7 @@ def check(self, context: SanityContext) -> list[Reason] | None: return [ Reason(f"File not found: {record['filename']}") for record in records - if record.get("is_valid", True) and not data_root.joinpath(record["filename"]).exists() + if record.get("is_valid", True) + and record["filename"] is not None # invalid (reported by FMT022), not a missing file + and not data_root.joinpath(record["filename"]).exists() ] or None diff --git a/t4_devkit/sanity/reference/ref204.py b/t4_devkit/sanity/reference/ref204.py new file mode 100644 index 00000000..1f50e8cd --- /dev/null +++ b/t4_devkit/sanity/reference/ref204.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from ..record.base import channel_file + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF204"] + + +@CHECKERS.register() +class REF204(RecordsChecker): + """A checker of REF204.""" + + id = RuleID("REF204") + name = RuleName("sample-data-file-of-sensor") + severity = Severity.ERROR + description = ( + "For a channel stored in one file, 'SampleData.filename' is its sensor's " + "'camera.video.path' / 'lidar.container.path'." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + files = { + sensor.token: stored[0] + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + if (stored := channel_file(sensor)) is not None + } + return [ + Reason( + f"SampleData {frame.token}: 'filename' {frame.filename}, sensor file {files[frame.sensor_token]}" + ) + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + if frame.sensor_token in files and frame.filename != files[frame.sensor_token] + ] or None diff --git a/t4_devkit/sanity/reference/ref205.py b/t4_devkit/sanity/reference/ref205.py new file mode 100644 index 00000000..91a00425 --- /dev/null +++ b/t4_devkit/sanity/reference/ref205.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .files import describe_lidar_file, describe_video_file + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF205"] + + +@CHECKERS.register() +class REF205(RecordsChecker): + """A checker of REF205.""" + + id = RuleID("REF205") + name = RuleName("sensor-file-description") + severity = Severity.ERROR + description = ( + "Each channel file matches its 'sensor' row: Parquet columns, dtypes and one row group per " + "frame; video codec, pixel format, bit depth, timescale, frame count and frame size." + ) + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + failures = [] + for sensor in context.load_records(SchemaName.SENSOR).unwrap(): + if sensor.camera is not None and sensor.camera.video is not None: + path = data_root.joinpath(sensor.camera.video.path) + if path.exists(): + failures += describe_video_file(sensor, path) + elif sensor.lidar is not None and sensor.lidar.container.path is not None: + path = data_root.joinpath(sensor.lidar.container.path) + if path.exists() and sensor.lidar.container.format.value == "parquet": + failures += describe_lidar_file(sensor, path) + return failures or None diff --git a/t4_devkit/sanity/reference/ref206.py b/t4_devkit/sanity/reference/ref206.py new file mode 100644 index 00000000..bfac4166 --- /dev/null +++ b/t4_devkit/sanity/reference/ref206.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pyarrow.parquet as pq + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from pathlib import Path + + from ..context import SanityContext + + +__all__ = ["REF206"] + + +@CHECKERS.register() +class REF206(RecordsChecker): + """A checker of REF206.""" + + id = RuleID("REF206") + name = RuleName("lidar-num-points") + severity = Severity.ERROR + description = "'SampleData.num_points' equals the number of points in its frame, when set." + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + files = { + sensor.token: data_root.joinpath(sensor.lidar.container.path) + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + if sensor.lidar is not None + and sensor.lidar.container.path is not None + and sensor.lidar.container.format.value == "parquet" + } + row_counts = { + token: _row_group_sizes(path) for token, path in files.items() if path.exists() + } + failures = [] + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + counts = row_counts.get(frame.sensor_token) + if counts is None or frame.num_points is None or frame.frame_index is None: + continue + if frame.frame_index >= len(counts) or counts[frame.frame_index] != frame.num_points: + failures.append( + Reason( + f"SampleData {frame.token}: 'num_points' {frame.num_points} differs from its frame" + ) + ) + return failures or None + + +def _row_group_sizes(path: Path) -> list[int]: + metadata = pq.ParquetFile(path).metadata + return [metadata.row_group(i).num_rows for i in range(metadata.num_row_groups)] diff --git a/t4_devkit/sanity/reference/ref207.py b/t4_devkit/sanity/reference/ref207.py new file mode 100644 index 00000000..15057dff --- /dev/null +++ b/t4_devkit/sanity/reference/ref207.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.common.io import file_sha256 +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF207"] + + +@CHECKERS.register() +class REF207(RecordsChecker): + """A checker of REF207.""" + + id = RuleID("REF207") + name = RuleName("map-file") + severity = Severity.ERROR + description = "'Map.filename' exists and has the SHA-256 'Map.sha256'." + tables = (SchemaName.MAP,) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + failures = [] + for map_ in context.load_records(SchemaName.MAP).unwrap(): + path = data_root.joinpath(map_.filename) + if not path.exists(): + failures.append(Reason(f"Map {map_.token}: '{map_.filename}' not found")) + elif file_sha256(path) != map_.sha256: + failures.append(Reason(f"Map {map_.token}: SHA-256 of '{map_.filename}' differs")) + return failures or None diff --git a/t4_devkit/sanity/reference/ref302.py b/t4_devkit/sanity/reference/ref302.py new file mode 100644 index 00000000..fe4faebb --- /dev/null +++ b/t4_devkit/sanity/reference/ref302.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing, Some + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF302"] + + +@CHECKERS.register() +class REF302(RecordsChecker): + """A checker of REF302.""" + + id = RuleID("REF302") + name = RuleName("instance-to-map-element") + severity = Severity.ERROR + description = "'Instance.map_element' refers to an element of its 'type' in the lanelet2 map." + tables = ( + SchemaName.MAP, + SchemaName.INSTANCE, + ) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match super().can_skip(context): + case Some(reason): + return Some(reason) + match context.lanelet_map: + case Some(_): + return Nothing + case _: + return Maybe.from_value(Reason("No lanelet2 map to check against")) + + def check(self, context: SanityContext) -> list[Reason] | None: + lanelet = context.lanelet_map.unwrap() + elements = { + "node": set(lanelet.nodes), + "way": set(lanelet.ways), + "relation": set(lanelet.relations), + } + return [ + Reason(f"Instance {instance.token}: no {element.type.value} {element.id} in the map") + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + if (element := instance.map_element).id is not None + and str(element.id) not in elements[element.type.value] + ] or None diff --git a/t4_devkit/sanity/reference/ref303.py b/t4_devkit/sanity/reference/ref303.py new file mode 100644 index 00000000..5fe66a32 --- /dev/null +++ b/t4_devkit/sanity/reference/ref303.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing, Some + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF303"] + + +@CHECKERS.register() +class REF303(RecordsChecker): + """A checker of REF303.""" + + id = RuleID("REF303") + name = RuleName("traffic-light-to-lanelet") + severity = Severity.ERROR + description = ( + "'TrafficLight.regulatory_element_id' is a traffic-light regulatory element of the " + "lanelet2 map." + ) + tables = ( + SchemaName.MAP, + SchemaName.TRAFFIC_LIGHT, + ) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match super().can_skip(context): + case Some(reason): + return Some(reason) + match context.lanelet_map: + case Some(_): + return Nothing + case _: + return Maybe.from_value(Reason("No lanelet2 map to check against")) + + def check(self, context: SanityContext) -> list[Reason] | None: + regulatory_elements = { + id + for id, relation in context.lanelet_map.unwrap().relations.items() + if relation.tags.get("type") == "regulatory_element" + and relation.tags.get("subtype") == "traffic_light" + } + return [ + Reason( + f"TrafficLight {light.token}: no traffic-light regulatory element {light.regulatory_element_id}" + ) + for light in context.load_records(SchemaName.TRAFFIC_LIGHT).unwrap() + if str(light.regulatory_element_id) not in regulatory_elements + ] or None diff --git a/t4_devkit/sanity/reference/ref304.py b/t4_devkit/sanity/reference/ref304.py new file mode 100644 index 00000000..6ca91b9a --- /dev/null +++ b/t4_devkit/sanity/reference/ref304.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing, Some + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF304"] + + +@CHECKERS.register() +class REF304(RecordsChecker): + """A checker of REF304.""" + + id = RuleID("REF304") + name = RuleName("route-to-lanelet") + severity = Severity.ERROR + description = "'Route' segment ids are lanelets of the lanelet2 map." + tables = ( + SchemaName.MAP, + SchemaName.ROUTE, + ) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match super().can_skip(context): + case Some(reason): + return Some(reason) + match context.lanelet_map: + case Some(_): + return Nothing + case _: + return Maybe.from_value(Reason("No lanelet2 map to check against")) + + def check(self, context: SanityContext) -> list[Reason] | None: + lanelets = { + id + for id, relation in context.lanelet_map.unwrap().relations.items() + if relation.tags.get("type") == "lanelet" + } + return [ + Reason(f"Route {route.token}: no lanelet {id}") + for route in context.load_records(SchemaName.ROUTE).unwrap() + for segment in route.segments + for id in sorted({segment.preferred_primitive_id, *segment.primitive_ids}) + if str(id) not in lanelets + ] or None diff --git a/t4_devkit/sanity/reference/ref305.py b/t4_devkit/sanity/reference/ref305.py new file mode 100644 index 00000000..c2c6bbad --- /dev/null +++ b/t4_devkit/sanity/reference/ref305.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .files import lidar_column_values + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF305"] + + +@CHECKERS.register() +class REF305(RecordsChecker): + """A checker of REF305.""" + + id = RuleID("REF305") + name = RuleName("lidar-source-index") + severity = Severity.ERROR + description = ( + "In a merged LiDAR file, 'source_index' values are the 'Sensor.index' of the channel's " + "'source_channels'." + ) + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = context.load_records(SchemaName.SENSOR).unwrap() + data_root = context.data_root.unwrap() + failures = [] + for sensor in sensors: + if sensor.lidar is None or not sensor.lidar.source_channels: + continue + path = sensor.lidar.container.path + if path is None or not data_root.joinpath(path).exists(): + continue + expected = {s.index for s in sensors if s.channel in sensor.lidar.source_channels} + values = lidar_column_values(data_root.joinpath(path), "source_index") + if values is None: + failures.append(Reason(f"Sensor {sensor.channel}: no 'source_index' column")) + elif not values <= expected: + failures.append( + Reason( + f"Sensor {sensor.channel}: 'source_index' values {sorted(values - expected)} are not its sources" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/reference/ref306.py b/t4_devkit/sanity/reference/ref306.py new file mode 100644 index 00000000..944df3fc --- /dev/null +++ b/t4_devkit/sanity/reference/ref306.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .files import lidar_column_values + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF306"] + + +@CHECKERS.register() +class REF306(RecordsChecker): + """A checker of REF306.""" + + id = RuleID("REF306") + name = RuleName("lidar-point-labels") + severity = Severity.ERROR + description = ( + "In a LiDAR file, 'label' values are 'Category.index' values, and 'instance' values are " + "'Instance.index' values or -1 (no object)." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.CATEGORY, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + indices = { + "label": {c.index for c in context.load_records(SchemaName.CATEGORY).unwrap()}, + "instance": {-1} + | {i.index for i in context.load_records(SchemaName.INSTANCE).unwrap()}, + } + failures = [] + for sensor in context.load_records(SchemaName.SENSOR).unwrap(): + if sensor.lidar is None or sensor.lidar.container.path is None: + continue + path = data_root.joinpath(sensor.lidar.container.path) + if not path.exists(): + continue + for column, known in indices.items(): + values = lidar_column_values(path, column) + if values is not None and not values <= known: + failures.append( + Reason( + f"Sensor {sensor.channel}: '{column}' values {sorted(values - known)} not defined" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/registry.py b/t4_devkit/sanity/registry.py index 322dbb0c..f3459307 100644 --- a/t4_devkit/sanity/registry.py +++ b/t4_devkit/sanity/registry.py @@ -4,7 +4,7 @@ from collections.abc import Callable, Sequence from enum import Enum, unique -from .checker import Checker, RuleID +from .checker import Checker, RuleID, SchemaVersion @unique @@ -69,11 +69,17 @@ def _add_module(self, module: type[Checker]) -> None: self[group][module.id] = module - def build(self, excludes: Sequence[str] | None = None) -> list[Checker]: + def build( + self, + excludes: Sequence[str] | None = None, + version: SchemaVersion | None = None, + ) -> list[Checker]: """Build a list of checkers from the registry. Args: excludes (Sequence[str] | None, optional): A list of rule IDs or rule groups to exclude. + version (SchemaVersion | None, optional): Keep only the rules of this schema version; + all rules if None. Returns: A list of checkers. @@ -85,7 +91,9 @@ def build(self, excludes: Sequence[str] | None = None) -> list[Checker]: checker() for group, values in self.items() for id, checker in values.items() - if id not in excludes and group.value not in excludes + if id not in excludes + and group.value not in excludes + and (version is None or version in checker.versions) ] diff --git a/t4_devkit/sanity/run.py b/t4_devkit/sanity/run.py index 30f0663b..3127caf9 100644 --- a/t4_devkit/sanity/run.py +++ b/t4_devkit/sanity/run.py @@ -20,6 +20,7 @@ def sanity_check( *, excludes: Sequence[str] | None = None, fix: bool = False, + checksum: bool = False, ) -> SanityResult: """Run sanity checks on the given data root. @@ -28,13 +29,16 @@ def sanity_check( revision (str | None, optional): The revision to check. If None, the latest revision is used. excludes (Sequence[str] | None, optional): A list of rule names or groups to exclude. fix (bool, optional): Attempt to fix the issues reported by the sanity check. + checksum (bool, optional): Verify the SHA-256 of every file listed in `manifest.json` + (1.0 only). Reads every file, including videos and rosbags. Returns: A SanityResult object. """ - context = SanityContext.from_path(data_root, revision=revision) + context = SanityContext.from_path(data_root, revision=revision, checksum=checksum) - checkers = CHECKERS.build(excludes=excludes) + # only the rules of the dataset's schema version + checkers = CHECKERS.build(excludes=excludes, version=context.schema_version) reports = [checker(context, fix=fix) for checker in checkers] return SanityResult.from_context(context, reports) diff --git a/t4_devkit/sanity/safety.py b/t4_devkit/sanity/safety.py index 8ec12807..b2f2b66c 100644 --- a/t4_devkit/sanity/safety.py +++ b/t4_devkit/sanity/safety.py @@ -6,6 +6,7 @@ from t4_devkit import DBMetadata, T4Devkit, load_metadata from t4_devkit.common import load_json, save_json +from t4_devkit.schema.v1 import Manifest if TYPE_CHECKING: from t4_devkit.schema import SchemaBase @@ -32,6 +33,12 @@ def load_schema_safe(module: type[SchemaBase], record: dict) -> SchemaBase: return module.from_dict(record) +@safe +def load_manifest_safe(data: dict) -> Manifest: + """Build the 1.0 manifest from its dict safely.""" + return Manifest(**data) + + @safe def load_metadata_safe(data_root: PathLike, revision: str | None = None) -> DBMetadata: """Load DBMetadata safely.""" diff --git a/t4_devkit/sanity/structure/__init__.py b/t4_devkit/sanity/structure/__init__.py index 0d1224d7..9430b102 100644 --- a/t4_devkit/sanity/structure/__init__.py +++ b/t4_devkit/sanity/structure/__init__.py @@ -9,3 +9,9 @@ from .str007 import * # noqa from .str008 import * # noqa from .str009 import * # noqa +from .str010 import * # noqa +from .str011 import * # noqa +from .str012 import * # noqa +from .str013 import * # noqa +from .str014 import * # noqa +from .str015 import * # noqa diff --git a/t4_devkit/sanity/structure/str001.py b/t4_devkit/sanity/structure/str001.py index 9894e6d1..8316ff77 100644 --- a/t4_devkit/sanity/structure/str001.py +++ b/t4_devkit/sanity/structure/str001.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR001(Checker): name = RuleName("version-dir-presence") severity = Severity.WARNING description = "'version/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.version: diff --git a/t4_devkit/sanity/structure/str002.py b/t4_devkit/sanity/structure/str002.py index 3a081faf..c4729eb0 100644 --- a/t4_devkit/sanity/structure/str002.py +++ b/t4_devkit/sanity/structure/str002.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR002(Checker): name = RuleName("annotation-dir-presence") severity = Severity.ERROR description = "'annotation/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.annotation_dir: diff --git a/t4_devkit/sanity/structure/str003.py b/t4_devkit/sanity/structure/str003.py index e83b3f16..6be6d209 100644 --- a/t4_devkit/sanity/structure/str003.py +++ b/t4_devkit/sanity/structure/str003.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR003(Checker): name = RuleName("data-dir-presence") severity = Severity.ERROR description = "'data/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.sensor_data_dir: diff --git a/t4_devkit/sanity/structure/str004.py b/t4_devkit/sanity/structure/str004.py index 2d0101ef..2d6a274f 100644 --- a/t4_devkit/sanity/structure/str004.py +++ b/t4_devkit/sanity/structure/str004.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR004(Checker): name = RuleName("map-dir-presence") severity = Severity.WARNING description = "'map/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: diff --git a/t4_devkit/sanity/structure/str005.py b/t4_devkit/sanity/structure/str005.py index 911ca56c..702a96bf 100644 --- a/t4_devkit/sanity/structure/str005.py +++ b/t4_devkit/sanity/structure/str005.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR005(Checker): name = RuleName("bag-dir-presence") severity = Severity.WARNING description = "'input_bag/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.bag_dir: diff --git a/t4_devkit/sanity/structure/str008.py b/t4_devkit/sanity/structure/str008.py index 95033016..0942bfc3 100644 --- a/t4_devkit/sanity/structure/str008.py +++ b/t4_devkit/sanity/structure/str008.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -22,6 +22,7 @@ class STR008(Checker): name = RuleName("lanelet-file-presence") severity = Severity.WARNING description = "'lanelet2_map.osm' file exists under the 'map/' directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: diff --git a/t4_devkit/sanity/structure/str009.py b/t4_devkit/sanity/structure/str009.py index 24ba4f40..77a8af43 100644 --- a/t4_devkit/sanity/structure/str009.py +++ b/t4_devkit/sanity/structure/str009.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -22,6 +22,7 @@ class STR009(Checker): name = RuleName("pointcloud-map-dir-presence") severity = Severity.WARNING description = "'pointcloud_map.pcd' directory exists under the 'map/' directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: diff --git a/t4_devkit/sanity/structure/str010.py b/t4_devkit/sanity/structure/str010.py new file mode 100644 index 00000000..17b29253 --- /dev/null +++ b/t4_devkit/sanity/structure/str010.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Some + +from t4_devkit.schema.v1 import MANIFEST_FILENAME, SUPPORTED_SCHEMA_MAJOR_VERSION + +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR010"] + + +@CHECKERS.register() +class STR010(Checker): + """A checker of STR010.""" + + id = RuleID("STR010") + name = RuleName("schema-version-supported") + severity = Severity.ERROR + description = ( + "'manifest.json' declares a supported 'schema_version' (major version 1). " + "If not, no other 1.0 rule runs." + ) + versions = (SchemaVersion.V1,) + + def check(self, context: SanityContext) -> list[Reason] | None: + match context.raw_manifest: + case Some(manifest): + version = manifest.get("schema_version") if isinstance(manifest, dict) else None + if not isinstance(version, str): + return [Reason(f"'schema_version' is missing or not a string: {version!r}")] + if version.split(".")[0] != str(SUPPORTED_SCHEMA_MAJOR_VERSION): + return [Reason(f"Unsupported 'schema_version': {version}")] + return None + return [Reason(f"'{MANIFEST_FILENAME}' cannot be read")] diff --git a/t4_devkit/sanity/structure/str011.py b/t4_devkit/sanity/structure/str011.py new file mode 100644 index 00000000..aed78f44 --- /dev/null +++ b/t4_devkit/sanity/structure/str011.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR011"] + + +@CHECKERS.register() +class STR011(ManifestChecker): + """A checker of STR011.""" + + id = RuleID("STR011") + name = RuleName("manifest-files-presence") + severity = Severity.ERROR + description = "Every file listed in 'manifest.json' ('tables' and 'assets') exists." + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + return [ + Reason(f"Listed file not found: {path}") + for path in context.manifest.unwrap().listed_paths() + if not data_root.joinpath(path).exists() + ] or None diff --git a/t4_devkit/sanity/structure/str012.py b/t4_devkit/sanity/structure/str012.py new file mode 100644 index 00000000..4c376b18 --- /dev/null +++ b/t4_devkit/sanity/structure/str012.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.common.io import file_sha256 +from t4_devkit.schema.v1 import AssetEntry + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason, make_report + +if TYPE_CHECKING: + from ..context import SanityContext + from ..result import Report + + +__all__ = ["STR012"] + + +@CHECKERS.register() +class STR012(ManifestChecker): + """A checker of STR012. + + Opt-in (`checksum=True`, `t4sanity --checksum`), since it reads every file, including videos + and rosbags. Without it, the rule reports a warning that hashes were not verified. + """ + + id = RuleID("STR012") + name = RuleName("manifest-files-checksum") + severity = Severity.ERROR + description = ( + "Every listed file has the listed 'sha256', and every asset the listed 'bytes'. " + "Opt-in ('--checksum'); a warning if not run." + ) + + def __call__(self, context: SanityContext, fix: bool = False) -> Report: + if not context.checksum: + reasons = [Reason("Hashes not verified: run with '--checksum'")] + return make_report(self.id, self.name, Severity.WARNING, self.description, reasons) + return super().__call__(context, fix=fix) + + def check(self, context: SanityContext) -> list[Reason] | None: + manifest = context.manifest.unwrap() + data_root = context.data_root.unwrap() + failures = [] + entries = [*manifest.tables.values(), *manifest.assets] + for entry in entries: + filepath = data_root.joinpath(entry.path) + if not filepath.exists(): + continue # reported by STR011 + if isinstance(entry, AssetEntry) and filepath.stat().st_size != entry.bytes: + failures.append( + Reason(f"{entry.path}: {filepath.stat().st_size} bytes, listed {entry.bytes}") + ) + elif file_sha256(filepath) != entry.sha256: + failures.append(Reason(f"{entry.path}: SHA-256 differs from 'manifest.json'")) + return failures or None diff --git a/t4_devkit/sanity/structure/str013.py b/t4_devkit/sanity/structure/str013.py new file mode 100644 index 00000000..4c7a6e01 --- /dev/null +++ b/t4_devkit/sanity/structure/str013.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import Manifest + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR013"] + + +@CHECKERS.register() +class STR013(ManifestChecker): + """A checker of STR013.""" + + id = RuleID("STR013") + name = RuleName("manifest-files-complete") + severity = Severity.WARNING + description = ( + "Every file under 'annotation/', 'data/', 'map/' and 'input_bag/' is listed in " + "'manifest.json'." + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + listed = set(context.manifest.unwrap().listed_paths()) + return [ + Reason(f"File not listed: {path}") + for directory in ("annotation", *Manifest.ASSET_DIRS) + for path in sorted( + filepath.relative_to(data_root).as_posix() + for filepath in data_root.joinpath(directory).rglob("*") + if filepath.is_file() + ) + if path not in listed + ] or None diff --git a/t4_devkit/sanity/structure/str014.py b/t4_devkit/sanity/structure/str014.py new file mode 100644 index 00000000..96ed1dfb --- /dev/null +++ b/t4_devkit/sanity/structure/str014.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR014"] + + +@CHECKERS.register() +class STR014(ManifestChecker): + """A checker of STR014.""" + + id = RuleID("STR014") + name = RuleName("manifest-required-tables") + severity = Severity.ERROR + description = "'manifest.json' lists every required table." + + def check(self, context: SanityContext) -> list[Reason] | None: + tables = context.manifest.unwrap().tables + return [ + Reason(f"Required table not listed: {table.value}") + for table in SchemaName + if not table.is_optional() and table.value not in tables + ] or None diff --git a/t4_devkit/sanity/structure/str015.py b/t4_devkit/sanity/structure/str015.py new file mode 100644 index 00000000..6107777d --- /dev/null +++ b/t4_devkit/sanity/structure/str015.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Some + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR015"] + + +@CHECKERS.register() +class STR015(ManifestChecker): + """A checker of STR015.""" + + id = RuleID("STR015") + name = RuleName("manifest-identity-matches-dirs") + severity = Severity.WARNING + description = ( + "'manifest.json' 'dataset_id' and 'version' match the names of the dataset and version " + "directories." + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + manifest = context.manifest.unwrap() + failures = [] + if manifest.dataset_id != context.dataset_id.unwrap(): + failures.append( + Reason( + f"'dataset_id' {manifest.dataset_id!r} differs from the directory " + f"{context.dataset_id.unwrap()!r}" + ) + ) + match context.version: + case Some(directory) if str(manifest.version) != directory: + failures.append( + Reason(f"'version' {manifest.version} differs from the directory {directory!r}") + ) + return failures or None diff --git a/t4_devkit/sanity/tier4/tiv001.py b/t4_devkit/sanity/tier4/tiv001.py index 25f46ebd..8a1a24c6 100644 --- a/t4_devkit/sanity/tier4/tiv001.py +++ b/t4_devkit/sanity/tier4/tiv001.py @@ -5,7 +5,7 @@ from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_tier4_safe @@ -24,6 +24,7 @@ class TIV001(Checker): name = RuleName("load-tier4") severity = Severity.ERROR description = "Ensure 'T4Devkit' instance is loaded successfully." + versions = (SchemaVersion.V0, SchemaVersion.V1) def can_skip(self, context: SanityContext) -> Maybe[Reason]: match context.data_root: diff --git a/t4_devkit/schema/compatibility.py b/t4_devkit/schema/compatibility.py deleted file mode 100644 index 86e0b8a2..00000000 --- a/t4_devkit/schema/compatibility.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Compatibility helpers that abstract away differences between T4Dataset revisions.""" - -from __future__ import annotations - -from t4_devkit.schema import Category - - -def fix_category_table(categories: list[Category]) -> list[Category]: - """Fix unpopulated index fields in some T4Dataset revisions. - - This function behaves differently in the below three cases: - - - All `index` fields are set: the list is returned unmodified. - - All `index` fields are `None`: the position of each category in the list is used to compute - `index`. The resulting indices start at `0`. - - Some `index` fields are set and some are `None`: raise a `ValueError`. - - Args: - categories (list[Category]): List of categories to fix. - - Returns: - list[Category]: Fixed list of categories. - - Raises: - ValueError: If the `index` field is set for some categories and `None` for others. - """ - if any(category.index is None for category in categories): - if not all(category.index is None for category in categories): - raise ValueError("Category index is not set for some categories.") - - for idx, category in enumerate(categories): - category.index = idx - return categories diff --git a/t4_devkit/schema/tables/registry.py b/t4_devkit/schema/tables/registry.py index 9f910cfb..121ef635 100644 --- a/t4_devkit/schema/tables/registry.py +++ b/t4_devkit/schema/tables/registry.py @@ -1,6 +1,7 @@ from __future__ import annotations import inspect +from enum import Enum from typing import TYPE_CHECKING, Callable, KeysView from ..name import SchemaName @@ -14,9 +15,14 @@ class SchemaRegistry: - """A manager class to register schema tables.""" + """A manager class to register schema tables. - def __init__(self) -> None: + Args: + name_type (type[Enum], optional): Enum of the table names this registry accepts. + """ + + def __init__(self, name_type: type[Enum] = SchemaName) -> None: + self.__name_type = name_type self.__schemas: dict[str, SchemaTable] = {} @property @@ -27,13 +33,13 @@ def __len__(self) -> int: return len(self.__schemas) def __contains__(self, key: str | SchemaName) -> bool: - if isinstance(key, SchemaName): + if isinstance(key, self.__name_type): key = key.value return key in self.keys() def get(self, key: str | SchemaName) -> SchemaTable: - if isinstance(key, SchemaName): + if isinstance(key, self.__name_type): key = key.value if key not in self.__schemas: @@ -44,8 +50,8 @@ def keys(self) -> KeysView[str]: return self.__schemas.keys() def register(self, name: SchemaName, *, force: bool = False) -> Callable: - if not isinstance(name, SchemaName): - raise TypeError(f"name must be SchemaName, but got {type(name)}.") + if not isinstance(name, self.__name_type): + raise TypeError(f"name must be {self.__name_type.__name__}, but got {type(name)}.") def _register_decorator(obj: object) -> object: self._add_module(obj, name=name, force=force) @@ -72,7 +78,7 @@ def build_from_json(self, key: str | SchemaName, filepath: PathLike) -> list[Sch Returns: Instantiated dataclass. """ - if isinstance(key, SchemaName): + if isinstance(key, self.__name_type): key = key.value schema: SchemaTable = self.get(key) diff --git a/t4_devkit/schema/v1/__init__.py b/t4_devkit/schema/v1/__init__.py new file mode 100644 index 00000000..0fca6121 --- /dev/null +++ b/t4_devkit/schema/v1/__init__.py @@ -0,0 +1,28 @@ +from .annotation_type import * # noqa +from .attribute import * # noqa +from .caption import * # noqa +from .category import * # noqa +from .ego_pose import * # noqa +from .instance import * # noqa +from .keypoint import * # noqa +from .loader import * # noqa +from .log import * # noqa +from .manifest import * # noqa +from .map import * # noqa +from .name import * # noqa +from .object_ann import * # noqa +from .registry import * # noqa +from .route import * # noqa +from .sample import * # noqa +from .sample_annotation import * # noqa +from .sample_data import * # noqa +from .scenario import * # noqa +from .scene import * # noqa +from .sensor import * # noqa +from .traffic_light import * # noqa +from .vehicle import * # noqa +from .vehicle_state import * # noqa +from .visibility import * # noqa +from .writer import * # noqa +from .convert_v0 import * # noqa +from .migrate_v0 import * # noqa diff --git a/t4_devkit/schema/v1/annotation_type.py b/t4_devkit/schema/v1/annotation_type.py new file mode 100644 index 00000000..6d0cd81f --- /dev/null +++ b/t4_devkit/schema/v1/annotation_type.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from enum import Enum, unique + +__all__ = ["AnnotationType"] + + +@unique +class AnnotationType(str, Enum): + """How a label was made. + + Every label table has `annotation_type`, `provenance` and `confidence` fields. + + Attributes: + MANUAL: Made by a human annotator. + AUTO: Made by an offline auto-labeling model or pipeline, not reviewed by a human. + PSEUDO: Taken from another system's output and used as a weak label, e.g. onboard + perception or tracker results. + CORRECTED: Made automatically (auto, pseudo or interpolated), then reviewed and + corrected by a human. + INTERPOLATED: Computed between labeled frames, not reviewed frame by frame. + SYNTHETIC: Known by construction, from simulation or generated data. + """ + + MANUAL = "manual" + AUTO = "auto" + PSEUDO = "pseudo" + CORRECTED = "corrected" + INTERPOLATED = "interpolated" + SYNTHETIC = "synthetic" diff --git a/t4_devkit/schema/v1/attribute.py b/t4_devkit/schema/v1/attribute.py new file mode 100644 index 00000000..cc6699d3 --- /dev/null +++ b/t4_devkit/schema/v1/attribute.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Attribute"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.ATTRIBUTE) +class Attribute(SchemaBase): + """`attribute.json` dataclass. + + Attributes: + token (str): Unique record identifier. + name (str): Attribute name. + description (str | None): Attribute description. + """ + + name: str = field(validator=validators.instance_of(str)) + description: str | None = field(validator=validators.optional(validators.instance_of(str))) diff --git a/t4_devkit/schema/v1/caption.py b/t4_devkit/schema/v1/caption.py new file mode 100644 index 00000000..791dae9d --- /dev/null +++ b/t4_devkit/schema/v1/caption.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .annotation_type import AnnotationType +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["CaptionType", "Caption"] + + +@unique +class CaptionType(str, Enum): + """Kind of caption text. + + Attributes: + DESCRIPTION: Describes what happens in the window. + REASONING: Explains why the ego vehicle or others act as they do. + INSTRUCTION: An instruction to the ego vehicle. + QUESTION_ANSWER: A question about the window and its answer. + """ + + DESCRIPTION = "description" + REASONING = "reasoning" + INSTRUCTION = "instruction" + QUESTION_ANSWER = "question_answer" + + +@define(slots=False) +@SCHEMAS.register(SchemaName.CAPTION) +class Caption(SchemaBase): + """`caption.json` dataclass. + + Natural-language text about a time window, for VLM/VLA training. + + Attributes: + token (str): Unique record identifier. + log_token (str): Foreign key pointing to the log. + start_timestamp (int): Start of the window, Unix time stamp [us]. + end_timestamp (int): End of the window [us]. Equal to `start_timestamp` for one moment. + sample_data_token (str | None): Foreign key pointing to the sample data the caption + describes, if it is about one image or sweep. + scenario_token (str | None): Foreign key pointing to a scenario. + instance_tokens (list[str] | None): Foreign keys pointing to the instances the text + refers to. + caption_type (CaptionType): Kind of text. + text (str): The text. + language (str): Language code, e.g. "en", "ja". + annotation_type (AnnotationType): How the label was made. + provenance (str | None): VLM name and version, prompt ID, annotator or vendor. + confidence (float | None): Confidence in [0, 1], or None for manual labels. + """ + + log_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + start_timestamp: int = field(validator=validators.instance_of(int)) + end_timestamp: int = field(validator=validators.instance_of(int)) + sample_data_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + scenario_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + instance_tokens: list[str] | None = field( + validator=validators.optional( + validators.deep_iterable( + validators.and_(validators.instance_of(str), impossible_empty()), + validators.instance_of(list), + ) + ) + ) + caption_type: CaptionType = field(converter=CaptionType) + text: str = field(validator=(validators.instance_of(str), impossible_empty())) + language: str = field(validator=(validators.instance_of(str), impossible_empty())) + annotation_type: AnnotationType = field(converter=AnnotationType) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) + + @end_timestamp.validator + def _check_end_timestamp(self, attribute, value: int) -> None: + if value < self.start_timestamp: + raise ValueError(f"end_timestamp {value} is before start_timestamp") diff --git a/t4_devkit/schema/v1/category.py b/t4_devkit/schema/v1/category.py new file mode 100644 index 00000000..d52479b4 --- /dev/null +++ b/t4_devkit/schema/v1/category.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Category"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.CATEGORY) +class Category(SchemaBase): + """`category.json` dataclass. + + Attributes: + token (str): Unique record identifier. + name (str): Category name. + index (int | None): Value of the LiDAR `label` field for this category. + description (str | None): Category description. + is_thing (bool): Whether annotations of this category are individual objects with an + instance (car, pedestrian) rather than regions without one (road, sky). + has_orientation (bool): Whether annotations of this category may have `orientation`. + has_number (bool): Whether annotations of this category may have `number`. + """ + + name: str = field(validator=(validators.instance_of(str), impossible_empty())) + index: int | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(int), validators.ge(0), validators.le(255)) + ) + ) + description: str | None = field(validator=validators.optional(validators.instance_of(str))) + is_thing: bool = field(validator=validators.instance_of(bool)) + has_orientation: bool = field(validator=validators.instance_of(bool)) + has_number: bool = field(validator=validators.instance_of(bool)) diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py new file mode 100644 index 00000000..bf906fe0 --- /dev/null +++ b/t4_devkit/schema/v1/convert_v0.py @@ -0,0 +1,904 @@ +from __future__ import annotations + +import base64 +import hashlib +import os.path as osp +import re +import warnings +from collections import defaultdict +from datetime import datetime, timezone +from glob import glob +from typing import TYPE_CHECKING, Any, ClassVar + +import pypcd4 +import yaml +from PIL import Image +from pycocotools import mask as cocomask + +from t4_devkit.common.io import V0_BIN_FIELDS, file_sha256, load_json +from t4_devkit.schema.tables.visibility import VisibilityLevel as V0VisibilityLevel + +from .instance import Instance +from .name import SchemaName +from .sensor import DistortionModel + +if TYPE_CHECKING: + from t4_devkit.typing import PathLike + +__all__ = ["V0Converter"] + +Record = dict[str, Any] + + +class V0Converter: + """Read a 0.x dataset as 1.0 table dicts, best effort. + + Only the table files are read (plus one header per LiDAR channel and the map files); sensor + data files are not rewritten, so frames keep one file each and `frame_index` is None. + Values that 0.x never stored come from `FALLBACKS`. + + Examples: + >>> tables = V0Converter(data_root).convert() + """ + + # Values 1.0 requires that 0.x never stored. These placeholders keep a converted dataset + # loadable, they are not facts about the dataset (listed in `docs/schema/v1/index.md`). + # `vehicle.model == "unknown"` marks placeholder vehicle dimensions. + FALLBACKS: ClassVar[dict[str, Any]] = { + "vehicle": { + "model": "unknown", + "length": 1.0, + "width": 1.0, + "height": 1.0, + "wheel_base": 1.0, + }, + "map": {"map_id": "unknown", "version_id": "unknown", "source": "webauto"}, + "map_filename": "map/lanelet2_map.osm", + "map_projector_filename": "map/map_projector_info.yaml", + "ego_pose": {"source": "localization", "frame": "map"}, + "lidar": {"frame_id": "base_link", "is_undistorted": True, "compression": "none"}, + "camera": {"is_rectified": True}, + } + + def __init__(self, data_root: PathLike) -> None: + """ + Args: + data_root (PathLike): Dataset root, i.e. the `/` directory. + """ + self.data_root = data_root + # Every 0.x table file present, by table name; a missing table reads as empty. + self.v0: dict[str, list[Record]] = defaultdict(list) + for filepath in glob(osp.join(data_root, "annotation", "*.json")): + self.v0[osp.splitext(osp.basename(filepath))[0]] = load_json(filepath) + + # Lookups shared by several tables. + self.sensors = {record["token"]: record for record in self.v0["sensor"]} + self.frames: dict[str, list[Record]] = defaultdict(list) + for record in self.v0["sample_data"]: + self.frames[record["calibrated_sensor_token"]].append(record) + self.thing_categories = { + record["category_token"] for record in self.v0["object_ann"] + self.v0["instance"] + } + self.surface_categories = {record["category_token"] for record in self.v0["surface_ann"]} + # A labeled frame is a key frame (0.x has a few labels on non-key frames). + self.labeled_frames = { + record["sample_data_token"] + for table in ("object_ann", "surface_ann", "keypoint") + for record in self.v0[table] + } + # 0.x samples are annotated frames only; 1.0 has one sample per frame. + self.samples, self.sample_of_frame, self.sample_remap = self._frame_samples() + self.infos: dict[str, list[Record]] = {} # filled by `_frame_infos` + self.traffic_light_ways = self._traffic_light_ways() + + def convert(self) -> dict[SchemaName, list[Record]]: + """Return every table converted to 1.0 records. + + Returns: + Raw 1.0 records per table; empty for tables not converted. + """ + v0 = self.v0 + maps = [self.convert_map(record) for record in v0["map"]] + maps = [record for record in maps if record is not None] + log2map = {token: record["token"] for record in maps for token in record["log_tokens"]} + visibilities = self.convert_visibilities() + known_visibility = {record["token"] for record in visibilities} + + tables: dict[SchemaName, list[Record]] = {name: [] for name in SchemaName} + tables[SchemaName.MAP] = maps + tables[SchemaName.VEHICLE] = [self.convert_vehicle(record) for record in v0["log"]] + tables[SchemaName.LOG] = [self.convert_log(record, log2map) for record in v0["log"]] + tables[SchemaName.SCENE] = [self.convert_scene(record) for record in v0["scene"]] + tables[SchemaName.SAMPLE] = self.samples + tables[SchemaName.EGO_POSE] = [self.convert_ego_pose(record) for record in v0["ego_pose"]] + tables[SchemaName.SENSOR] = [ + self.convert_sensor(index, record) + for index, record in enumerate(v0["calibrated_sensor"]) + ] + tables[SchemaName.SAMPLE_DATA] = [ + self.convert_sample_data(record) for record in v0["sample_data"] + ] + + tables[SchemaName.CATEGORY] = self.convert_categories() + tables[SchemaName.ATTRIBUTE] = [ + self.convert_attribute(record) for record in v0["attribute"] + ] + tables[SchemaName.VISIBILITY] = visibilities + tables[SchemaName.SAMPLE_ANNOTATION] = [ + self.convert_sample_annotation(record, known_visibility) + for record in v0["sample_annotation"] + ] + tables[SchemaName.OBJECT_ANN] = [ + self.convert_object_ann(record) for record in v0["object_ann"] + ] + [self.convert_surface_ann(record) for record in v0["surface_ann"]] + tables[SchemaName.KEYPOINT] = [self.convert_keypoint(record) for record in v0["keypoint"]] + tables[SchemaName.VEHICLE_STATE] = [ + self.convert_vehicle_state(record) for record in v0["vehicle_state"] + ] + # after the labels: an instance's count and first / last label are over all its labels + annotations = self._instance_annotations(tables) + tables[SchemaName.INSTANCE] = [ + self.convert_instance(record, annotations) for record in v0["instance"] + ] + return tables + + # --- core tables --- + + def convert_map(self, record: Record) -> Record | None: + """Return the 1.0 `map` row of a 0.x map, or None if the map file is missing. + + `sha256` comes from the map file, `projector` from its yaml. + + Args: + record (Record): 0.x `map` row. + + Returns: + The 1.0 row, or None. + """ + filename = record["filename"] or self.FALLBACKS["map_filename"] + filepath = osp.join(self.data_root, filename) + if not osp.exists(filepath): + return None + sha256 = file_sha256(filepath) + fallbacks = self.FALLBACKS["map"] + return { + "token": record["token"], + "name": None, + "log_tokens": record["log_tokens"], + "map_id": fallbacks["map_id"], + "version_id": fallbacks["version_id"], + "source": fallbacks["source"], + "format": "lanelet2_osm", + "filename": filename, + "sha256": sha256, + "projector": self._read_projector(), + } + + def convert_vehicle(self, log: Record) -> Record: + """Return the vehicle of a log: named after the 0.x `log.vehicle`, with placeholder + dimensions. + + 0.x has no vehicle table, so there is one vehicle per log. + + Args: + log (Record): 0.x `log` row. + + Returns: + The 1.0 `vehicle` row. + """ + fallbacks = self.FALLBACKS["vehicle"] + return { + "token": self._vehicle_token(log), + "name": self._none_if_empty(log["vehicle"]), + "model": fallbacks["model"], + "length": fallbacks["length"], + "width": fallbacks["width"], + "height": fallbacks["height"], + "wheel_base": fallbacks["wheel_base"], + "rear_axle_to_center": None, + "sensor_rig_id": None, + } + + def convert_log(self, record: Record, log2map: dict[str, str]) -> Record: + """Return the 1.0 `log` row of a 0.x log. + + Args: + record (Record): 0.x `log` row. + log2map (dict[str, str]): Map token of each log token. + + Returns: + The 1.0 row. + """ + return { + "token": record["token"], + "logfile": self._none_if_empty(record["logfile"]), + "vehicle_token": self._vehicle_token(record), + "data_captured": record["data_captured"] or self._capture_time(record["token"]), + "location": self._none_if_empty(record["location"]), + "map_token": log2map.get(record["token"]), + } + + def convert_scene(self, record: Record) -> Record: + """Return the 1.0 `scene` row, with counts and first / last sample over the 1.0 samples. + + Args: + record (Record): 0.x `scene` row. + + Returns: + The 1.0 row. + """ + samples = [sample for sample in self.samples if sample["scene_token"] == record["token"]] + return { + "token": record["token"], + "name": record["name"], + "description": self._none_if_empty(record["description"]), + "log_token": record["log_token"], + "nbr_samples": len(samples), + "first_sample_token": next(s["token"] for s in samples if s["prev"] is None), + "last_sample_token": next(s["token"] for s in samples if s["next"] is None), + } + + def convert_ego_pose(self, record: Record) -> Record: + """Return the 1.0 `ego_pose` row of a 0.x ego pose. + + Args: + record (Record): 0.x `ego_pose` row. + + Returns: + The 1.0 row. + """ + fallbacks = self.FALLBACKS["ego_pose"] + return { + "token": record["token"], + "translation": record["translation"], + "rotation": record["rotation"], + "timestamp": record["timestamp"], + "twist": record.get("twist"), + # 0.x acceleration is a raw IMU reading whose frame and gravity sign vary by + # vehicle; 1.0 acceleration is gravity-free. Both come from the backfill. + "acceleration": None, + "imu": None, + "geocoordinate": record.get("geocoordinate"), + "covariance": None, + "source": fallbacks["source"], + "frame": fallbacks["frame"], + } + + def convert_sensor(self, index: int, calibration: Record) -> Record: + """Return the 1.0 `sensor` row of a 0.x calibration, merged with its 0.x `sensor` row. + + One row per calibration, keeping the `calibrated_sensor` token, indexed in file order. + + Args: + index (int): Sensor index, the position of the calibration in the file. + calibration (Record): 0.x `calibrated_sensor` row. + + Returns: + The 1.0 row. + """ + sensor = self.sensors[calibration["sensor_token"]] + modality = sensor["modality"] + return { + "token": calibration["token"], + "channel": sensor["channel"], + "index": index, + "modality": modality, + "translation": calibration["translation"], + "rotation": calibration["rotation"], + "camera": self.convert_camera(calibration) if modality == "camera" else None, + "lidar": self.convert_lidar(calibration) if modality == "lidar" else None, + "radar": None, + "hardware": None, + } + + def convert_camera(self, calibration: Record) -> Record: + """Return the camera block of a calibration: size from `sample_data`, or from the first + image if 0.x stored none. + + Args: + calibration (Record): 0.x `calibrated_sensor` row of a camera. + + Returns: + The camera block. + """ + distortion = calibration["camera_distortion"] + distortion_model = DistortionModel.from_num_coefficients(len(distortion)) + frames = self.frames[calibration["token"]] + sized = [record for record in frames if record.get("width") and record.get("height")] + if sized: + width, height = sized[0]["width"], sized[0]["height"] + else: + with Image.open(osp.join(self.data_root, frames[0]["filename"])) as image: + width, height = image.size + return { + "width": width, + "height": height, + "intrinsic": calibration["camera_intrinsic"], + "distortion": distortion, + "distortion_model": distortion_model.value, + "is_rectified": self.FALLBACKS["camera"]["is_rectified"], + "video": None, + } + + def convert_lidar(self, calibration: Record) -> Record: + """Return the LiDAR block of a calibration, from its first frame: its format, its `.info` + file (if any) and its header. + + Args: + calibration (Record): 0.x `calibrated_sensor` row of a LiDAR. + + Returns: + The LiDAR block. + """ + frames = self.frames[calibration["token"]] + if not frames: + return self._convert_source_lidar(calibration) + first = frames[0] + info = self._frame_infos(calibration["token"])[0] + if first["fileformat"] == "pcd": + filepath = osp.join(self.data_root, first["filename"]) + data = pypcd4.PointCloud.from_path(filepath).pc_data + container_format = "pcd" + fields = [{"name": name, "dtype": str(data.dtype[name])} for name in data.dtype.names] + else: + container_format = "bin" + num_pts_feats = info.get("num_pts_feats", 5) + fields = [{"name": name, "dtype": "float32"} for name in V0_BIN_FIELDS[:num_pts_feats]] + fallbacks = self.FALLBACKS["lidar"] + return { + "container": { + "format": container_format, + "path": None, + "frame_count": len(frames), + "compression": fallbacks["compression"], + }, + "fields": fields, + "source_channels": [ + self.sensors[source["sensor_token"]]["channel"] + for source in info.get("sources", []) + ], + "frame_id": fallbacks["frame_id"], + "is_undistorted": fallbacks["is_undistorted"], + } + + def convert_sample_data(self, record: Record) -> Record: + """Return the 1.0 `sample_data` row of a 0.x frame, in the 1.0 sample of its frame. + + Args: + record (Record): 0.x `sample_data` row. + + Returns: + The 1.0 row. + """ + return { + "token": record["token"], + "sample_token": self.sample_of_frame[record["token"]], + "ego_pose_token": record["ego_pose_token"], + # 0.x calibrated_sensor rows become 1.0 sensor rows, with the same tokens. + "sensor_token": record["calibrated_sensor_token"], + "filename": record["filename"], + "frame_index": None, + "fileformat": record["fileformat"], + "num_points": None, + "timestamp": record["timestamp"], + "is_key_frame": self._is_key_frame(record), + "is_valid": record.get("is_valid", True), + "next": self._none_if_empty(record["next"]), + "prev": self._none_if_empty(record["prev"]), + } + + # --- label tables --- + + def convert_categories(self) -> list[Record]: + """Return the 1.0 `category` rows. Categories without `index` get their list position, as + the 0.x devkit reads them. + + Returns: + The 1.0 rows. + + Raises: + ValueError: If some categories have an `index` and others do not. + """ + records = self.v0["category"] + indices = [record.get("index") for record in records] + if None in indices: + if any(index is not None for index in indices): + raise ValueError("Category index is not set for some categories.") + indices = list(range(len(records))) + return [self.convert_category(record, index) for record, index in zip(records, indices)] + + def convert_category(self, record: Record, index: int) -> Record: + """Return the 1.0 `category` row. `is_thing` is false only for categories used by + `surface_ann` (road, sky) and nothing else. + + Args: + record (Record): 0.x `category` row. + index (int): Category index. + + Returns: + The 1.0 row. + """ + token = record["token"] + return { + "token": token, + "name": record["name"], + "index": index, + "description": self._none_if_empty(record.get("description")), + "is_thing": token in self.thing_categories or token not in self.surface_categories, + "has_orientation": record.get("has_orientation", False), + "has_number": record.get("has_number", False), + } + + def convert_attribute(self, record: Record) -> Record: + """Return the 1.0 `attribute` row of a 0.x attribute. + + Args: + record (Record): 0.x `attribute` row. + + Returns: + The 1.0 row. + """ + return { + "token": record["token"], + "name": record["name"], + "description": self._none_if_empty(record.get("description")), + } + + def convert_visibilities(self) -> list[Record]: + """Return the 1.0 `visibility` rows: levels as read by 0.x (aliases mapped); `unavailable` + rows are dropped (visibility unknown). + + Returns: + The 1.0 rows. + """ + rows = [] + for record in self.v0["visibility"]: + level = V0VisibilityLevel.from_value(record["level"]) + if level != V0VisibilityLevel.UNAVAILABLE: + rows.append( + { + "token": record["token"], + "level": level.value, + "description": self._none_if_empty(record.get("description")), + } + ) + return rows + + def convert_instance( + self, record: Record, annotations: dict[str, tuple[int, str, str]] + ) -> Record: + """Return the 1.0 `instance` row. A lamp listed in 0.x traffic_light points to its lanelet2 + way in `map_element`. + + `nbr_annotations` and the first / last annotation come from `annotations`: 1.0 counts 3D + boxes and 2D labels, and 0.x tools count them differently (some write 0 for 2D). + + Args: + record (Record): 0.x `instance` row. + annotations (dict[str, tuple[int, str, str]]): Label count and first / last label token + of each instance (`Instance.annotation_spans`). + + Returns: + The 1.0 row. + """ + way_id = self.traffic_light_ways.get(record["token"]) + count, first, last = annotations.get(record["token"], (0, None, None)) + return { + "token": record["token"], + "name": record["instance_name"], + "index": None, + "category_token": record["category_token"], + "nbr_annotations": count, + "first_annotation_token": first, + "last_annotation_token": last, + "map_element": {"id": way_id, "type": "way" if way_id is not None else None}, + } + + def convert_sample_annotation(self, record: Record, known_visibility: set[str]) -> Record: + """Return the 1.0 `sample_annotation` row. Visibility levels dropped by + `convert_visibilities` become a null `visibility_token`. + + Args: + record (Record): 0.x `sample_annotation` row. + known_visibility (set[str]): Tokens of the visibility levels kept. + + Returns: + The 1.0 row. + """ + visibility_token = record["visibility_token"] + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_token": self.sample_remap.get(record["sample_token"], record["sample_token"]), + "instance_token": record["instance_token"], + "attribute_tokens": record["attribute_tokens"], + "visibility_token": visibility_token if visibility_token in known_visibility else None, + "translation": record["translation"], + "size": record["size"], + "rotation": record["rotation"], + "velocity": record.get("velocity"), + "acceleration": record.get("acceleration"), + "num_lidar_pts": record["num_lidar_pts"], + "num_radar_pts": record["num_radar_pts"], + "next": self._none_if_empty(record["next"]), + "prev": self._none_if_empty(record["prev"]), + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + def convert_object_ann(self, record: Record) -> Record: + """Return the 1.0 `object_ann` row of a 0.x 2D label. + + Args: + record (Record): 0.x `object_ann` row. + + Returns: + The 1.0 row. + """ + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_data_token": record["sample_data_token"], + "instance_token": record["instance_token"], + "category_token": record["category_token"], + "attribute_tokens": record["attribute_tokens"], + # 0.x leaves bbox out for mask-only labels; 1.0 requires it. + "bbox": record.get("bbox") or self._mask_bbox(record["mask"]), + "mask": self._non_empty_mask(record.get("mask")), + "orientation": record.get("orientation"), + "number": record.get("number"), + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + def convert_surface_ann(self, record: Record) -> Record: + """Return a 0.x semantic mask as an `object_ann` row without instance; its box is the + mask's. + + Args: + record (Record): 0.x `surface_ann` row. + + Returns: + The 1.0 `object_ann` row. + """ + mask = record["mask"] + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_data_token": record["sample_data_token"], + "instance_token": None, + "category_token": record["category_token"], + "attribute_tokens": [], + "bbox": self._mask_bbox(mask), + "mask": mask, + "orientation": None, + "number": None, + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + def convert_keypoint(self, record: Record) -> Record: + """Return the 1.0 `keypoint` row of a 0.x keypoint label. + + Args: + record (Record): 0.x `keypoint` row. + + Returns: + The 1.0 row. + """ + annotation_type, provenance, confidence = self._label_origin(record) + return { + "token": record["token"], + "sample_data_token": record["sample_data_token"], + "instance_token": record["instance_token"], + "category_tokens": record["category_tokens"], + "keypoints": record["keypoints"], + "num_keypoints": record["num_keypoints"], + "annotation_type": annotation_type, + "provenance": provenance, + "confidence": confidence, + } + + # --- planning tables --- + + def convert_vehicle_state(self, record: Record) -> Record: + """Return the 1.0 `vehicle_state` row. 0.x vehicle states come from the Autoware vehicle + interface (`/vehicle/status/*`). + + `additional_info.speed` (norm of the longitudinal and lateral velocity) is dropped: it can't + be split back into the 1.0 velocities. + + Args: + record (Record): 0.x `vehicle_state` row. + + Returns: + The 1.0 row. + """ + shift_state = record.get("shift_state") + # The 0.x devkit names forward gears FORWARD or HIGH; the 0.x converter writes DRIVE. + if shift_state in ("FORWARD", "HIGH"): + shift_state = "DRIVE" + return { + "token": record["token"], + "timestamp": record["timestamp"], + "accel_pedal": record.get("accel_pedal"), + "brake_pedal": record.get("brake_pedal"), + "steer_pedal": record.get("steer_pedal"), + "steering_tire_angle": record.get("steering_tire_angle"), + "steering_wheel_angle": record.get("steering_wheel_angle"), + "shift_state": shift_state, + "indicators": record.get("indicators"), + "longitudinal_velocity": None, + "lateral_velocity": None, + "heading_rate": None, + "source": "vehicle_interface", + } + + # --- helpers --- + + def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: + """Return one 1.0 sample per 0.x frame: the frames of all channels with the same frame + number. + + A frame with 0.x key samples keeps the earliest one's token; camera-only 0.x datasets have + one sample per camera image, merged here. Other frames get a new deterministic token. + + Returns: + The sample rows (chained by `next` / `prev` per scene), the sample token of each + `sample_data` token, and the token kept for each merged 0.x sample. + """ + v0_samples = {record["token"]: record for record in self.v0["sample"]} + lidars = { + calibration["token"] + for calibration in self.v0["calibrated_sensor"] + if self.sensors[calibration["sensor_token"]]["modality"] == "lidar" + } + + rows: list[Record] = [] + sample_of_frame: dict[str, str] = {} + remap: dict[str, str] = {} + for frames in self._frame_groups().values(): + key_samples = sorted( + { + frame["sample_token"] + for frame in frames + if frame["is_key_frame"] and frame["sample_token"] in v0_samples + }, + key=lambda token: (v0_samples[token]["timestamp"], token), + ) + if key_samples: + token = key_samples[0] + remap.update({merged: token for merged in key_samples}) + else: + first = min(frame["token"] for frame in frames) + token = hashlib.md5(f"{first}/sample".encode()).hexdigest() + # 0.x non-key frames point to the next sample: same scene + linked = [ + frame["sample_token"] for frame in frames if frame["sample_token"] in v0_samples + ] + scene_token = ( + v0_samples[linked[0]]["scene_token"] if linked else self.v0["scene"][0]["token"] + ) + lidar_frames = [frame for frame in frames if frame["calibrated_sensor_token"] in lidars] + rows.append( + { + "token": token, + "timestamp": min(frame["timestamp"] for frame in lidar_frames or frames), + "scene_token": scene_token, + "has_key_frame": any(self._is_key_frame(frame) for frame in frames), + "next": None, + "prev": None, + } + ) + sample_of_frame.update({frame["token"]: token for frame in frames}) + + by_scene: dict[str, list[Record]] = defaultdict(list) + for row in sorted(rows, key=lambda row: (row["timestamp"], row["token"])): + by_scene[row["scene_token"]].append(row) + for chain in by_scene.values(): + for row, following in zip(chain, chain[1:]): + row["next"], following["prev"] = following["token"], row["token"] + return [row for chain in by_scene.values() for row in chain], sample_of_frame, remap + + def _frame_groups(self) -> dict[int, list[Record]]: + """Return 0.x `sample_data` grouped by frame number (the digits starting the file name), or + by position in each channel's frames in time if a file name has no number. + """ + # The 0.x converter names the files of one frame by its frame number, e.g. `00042.jpg` + # and `00042.pcd.bin`: it groups the channels of a frame (their timestamps differ by up + # to ~90 ms). + frame_number = re.compile(r"^(\d+)\.") + records = self.v0["sample_data"] + numbers = [frame_number.match(osp.basename(record["filename"])) for record in records] + groups: dict[int, list[Record]] = defaultdict(list) + if all(numbers): + for record, number in zip(records, numbers): + groups[int(number.group(1))].append(record) + else: + for frames in self.frames.values(): + for position, record in enumerate(sorted(frames, key=lambda r: r["timestamp"])): + groups[position].append(record) + return groups + + def _is_key_frame(self, record: Record) -> bool: + """Return whether a 0.x frame is a key frame: marked as one, or carrying a 2D label or + keypoint. + """ + return record["is_key_frame"] or record["token"] in self.labeled_frames + + def _instance_annotations( + self, tables: dict[SchemaName, list[Record]] + ) -> dict[str, tuple[int, str, str]]: + """Return the number of labels and the first / last label token in time of each instance + (`Instance.annotation_spans`).""" + sample_times = { + sample["token"]: sample["timestamp"] for sample in tables[SchemaName.SAMPLE] + } + frame_times = { + frame["token"]: frame["timestamp"] for frame in tables[SchemaName.SAMPLE_DATA] + } + boxes = [ + (ann["instance_token"], sample_times[ann["sample_token"]], ann["token"]) + for ann in tables[SchemaName.SAMPLE_ANNOTATION] + ] + labels_2d = [ + (ann["instance_token"], frame_times[ann["sample_data_token"]], ann["token"]) + for ann in tables[SchemaName.OBJECT_ANN] + if ann["instance_token"] is not None + ] + return Instance.annotation_spans(boxes + labels_2d) + + def _read_projector(self) -> Record: + """Return the `projector` block from Autoware's `map_projector_info.yaml` (empty values + without it). + """ + filepath = osp.join(self.data_root, self.FALLBACKS["map_projector_filename"]) + info = {} + if osp.exists(filepath): + with open(filepath) as f: + info = yaml.safe_load(f) or {} + origin = info.get("map_origin") or {} + return { + "type": info.get("projector_type"), + "mgrs_grid": info.get("mgrs_grid"), + "origin": { + "lat": origin.get("latitude"), + "lon": origin.get("longitude"), + "alt": origin.get("altitude"), + }, + } + + def _convert_source_lidar(self, calibration: Record) -> Record: + """Return the LiDAR block of a LiDAR without frames of its own: a source of a concatenated + channel. + + Its points are stored in the concatenated files, so it has their format and fields; it + counts a frame wherever the `.info` file of a concatenated frame lists points from it. + """ + sensor_token = calibration["sensor_token"] + fallbacks = self.FALLBACKS["lidar"] + container_format, fields, frame_count = "bin", [], 0 + for concat in self.v0["calibrated_sensor"]: + infos = self._frame_infos(concat["token"]) + lengths = [ + sum( + s["length"] + for s in info.get("sources", []) + if s["sensor_token"] == sensor_token + ) + for info in infos + ] + if any(lengths): + concat_lidar = self.convert_lidar(concat) + container_format = concat_lidar["container"]["format"] + fields = concat_lidar["fields"] + frame_count = sum(length > 0 for length in lengths) + break + return { + "container": { + "format": container_format, + "path": None, + "frame_count": frame_count, + "compression": fallbacks["compression"], + }, + "fields": fields, + "source_channels": [], + "frame_id": fallbacks["frame_id"], + "is_undistorted": fallbacks["is_undistorted"], + } + + def _capture_time(self, log_token: str) -> str: + """Return the log's first sample time, ISO 8601 UTC, for logs without `data_captured`.""" + scenes = { + record["token"] for record in self.v0["scene"] if record["log_token"] == log_token + } + timestamp = min( + record["timestamp"] for record in self.v0["sample"] if record["scene_token"] in scenes + ) + return datetime.fromtimestamp(timestamp / 1e6, tz=timezone.utc).isoformat() + + def _traffic_light_ways(self) -> dict[str, int]: + """Return the lanelet2 way (`type=traffic_light`) each lamp instance shows, by instance + token. + + Two 0.x tables hold this link: `traffic_light` (`primitive_id`, #288) and + `traffic_light_instance_map` (`traffic_light_linestring_id`, from the unmerged + `feat/traffic-light-instance-map` branch), which wins when both list a lamp. `traffic_light` + rows without `primitive_id` are skipped, as no 0.x devkit reads them. + """ + ways = {} + skipped = 0 + for record in self.v0["traffic_light"]: + if "primitive_id" in record: + ways[record["instance_token"]] = int(record["primitive_id"]) + else: + skipped += 1 + if skipped: + warnings.warn(f"Skipped {skipped} traffic_light rows without primitive_id") + for record in self.v0["traffic_light_instance_map"]: + ways[record["instance_token"]] = int(record["traffic_light_linestring_id"]) + return ways + + def _frame_infos(self, calibration_token: str) -> list[Record]: + """Return the 0.x `.info` file of each frame of a channel ({} where there is none), read + once. + """ + if calibration_token not in self.infos: + self.infos[calibration_token] = [ + load_json(osp.join(self.data_root, frame["info_filename"])) + if frame.get("info_filename") + and osp.exists(osp.join(self.data_root, frame["info_filename"])) + else {} + for frame in self.frames[calibration_token] + ] + return self.infos[calibration_token] + + @staticmethod + def _non_empty_mask(mask: Record | None) -> Record | None: + """Return the mask, or None for a mask without pixels. + + Some 0.x tools give box-only labels a placeholder: an empty RLE whose `size` is (width, + height) instead of the (height, width) of real masks. + """ + if mask is None or "counts" not in mask: + return None + rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} + return mask if cocomask.area(rle) > 0 else None + + @staticmethod + def _mask_bbox(mask: Record) -> list[float]: + """Return the box around a 0.x RLE mask, as (xmin, ymin, xmax, ymax) on pixel edges. + + Same convention as 0.x boxes, which the 0.x converter writes as (x, y, x + w, y + h) clipped + to the image size. Mask `size` is (height, width), as in COCO. + """ + rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} + x, y, w, h = (float(v) for v in cocomask.toBbox(rle)) + return [x, y, x + w, y + h] + + @staticmethod + def _vehicle_token(log: Record) -> str: + """Return a deterministic vehicle token for a log: 0.x has no vehicle table.""" + return hashlib.md5(f"{log['token']}/vehicle".encode()).hexdigest() + + @staticmethod + def _none_if_empty(value: Any) -> Any: + """Return None for "", which 0.x writes for missing values; other values unchanged.""" + return None if value == "" else value + + @staticmethod + def _label_origin(record: Record) -> tuple[str, str | None, float | None]: + """Return `annotation_type`, `provenance` and `confidence` from the 0.x autolabel fields. + + Manual unless 0.x marks the label as automatic. `provenance` lists the autolabel models; + `confidence` is the model score when there is exactly one model. + """ + if not record.get("automatic_annotation", False): + return "manual", None, None + models = record.get("autolabel_metadata") or [] + provenance = ", ".join(model["name"] for model in models) or None + confidence = models[0]["score"] if len(models) == 1 else None + return "auto", provenance, confidence diff --git a/t4_devkit/schema/v1/ego_pose.py b/t4_devkit/schema/v1/ego_pose.py new file mode 100644 index 00000000..9250ddf4 --- /dev/null +++ b/t4_devkit/schema/v1/ego_pose.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import converters, define, field, validators + +from t4_devkit.common.converter import to_quaternion +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty +from t4_devkit.typing import Quaternion, Vector3, Vector6 + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["EgoPoseSource", "EgoPose"] + + +@unique +class EgoPoseSource(str, Enum): + """Where an ego pose came from.""" + + LOCALIZATION = "localization" + POST_PROCESSED = "post_processed" + GNSS_INS = "gnss_ins" + + +@define(slots=False) +@SCHEMAS.register(SchemaName.EGO_POSE) +class EgoPose(SchemaBase): + """`ego_pose.json` dataclass. + + Attributes: + token (str): Unique record identifier. + translation (Vector3): Position in `frame`, given as [x, y, z] in [m]. + rotation (Quaternion): Orientation in `frame`, given as quaternion [w, x, y, z]. + timestamp (int): Unix time stamp [us]. + twist (Vector6 | None): (vx, vy, vz, yaw_rate, pitch_rate, roll_rate) in the vehicle frame. + acceleration (Vector3 | None): (ax, ay, az) in the vehicle frame [m/s^2], without gravity: + the localization estimate, what Autoware feeds the planner. + imu (Vector6 | None): (ax, ay, az, wx, wy, wz): the raw IMU in the vehicle frame, specific + force with gravity (az ≈ +9.8 m/s^2 at rest) and angular velocity [rad/s]. + geocoordinate (Vector3 | None): (latitude, longitude, altitude). + source (EgoPoseSource): Where the pose came from. + frame (str): Coordinate frame of the pose, e.g. "map". + covariance (list[float] | None): 6x6 pose covariance, row-major. + + `twist`, `acceleration`, `imu`, `geocoordinate` and `covariance` hold the latest message at + or before `timestamp`, best effort; the pose itself is interpolated at `timestamp`. + """ + + translation: Vector3 = field(converter=Vector3) + rotation: Quaternion = field(converter=to_quaternion) + timestamp: int = field(validator=validators.instance_of(int)) + twist: Vector6 | None = field(converter=converters.optional(Vector6)) + acceleration: Vector3 | None = field(converter=converters.optional(Vector3)) + imu: Vector6 | None = field(converter=converters.optional(Vector6)) + geocoordinate: Vector3 | None = field(converter=converters.optional(Vector3)) + source: EgoPoseSource = field(converter=EgoPoseSource) + frame: str = field(validator=(validators.instance_of(str), impossible_empty())) + covariance: list[float] | None = field( + validator=validators.optional( + validators.deep_iterable( + validators.instance_of((int, float)), + (validators.instance_of(list), validators.min_len(36), validators.max_len(36)), + ) + ) + ) diff --git a/t4_devkit/schema/v1/instance.py b/t4_devkit/schema/v1/instance.py new file mode 100644 index 00000000..641127cf --- /dev/null +++ b/t4_devkit/schema/v1/instance.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import Iterable + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .map import MapElement +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Instance"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.INSTANCE) +class Instance(SchemaBase): + """`instance.json` dataclass. + + Attributes: + token (str): Unique record identifier. + name (str): `::`. + index (int | None): Value of the LiDAR `instance` field (int16) for this object, + 0 to 32767. + category_token (str): Foreign key pointing to the category. + nbr_annotations (int): Number of annotations of this instance. + first_annotation_token (str | None): Foreign key pointing to the first sample annotation + or object annotation. + last_annotation_token (str | None): Foreign key pointing to the last sample annotation + or object annotation. + map_element (MapElement): Link to a lanelet2 element. + """ + + name: str = field(validator=validators.instance_of(str)) + index: int | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(int), validators.ge(0), validators.le(2**15 - 1)) + ) + ) + category_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + nbr_annotations: int = field(validator=(validators.instance_of(int), validators.ge(0))) + first_annotation_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + last_annotation_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + map_element: MapElement = field( + converter=lambda x: MapElement(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(MapElement), + ) + + @staticmethod + def annotation_spans( + labels: Iterable[tuple[str, int, str]], + ) -> dict[str, tuple[int, str, str]]: + """Return the number of labels and the first / last label in time of each instance. + + This is what `nbr_annotations` and `first_annotation_token` / `last_annotation_token` + hold. + + Args: + labels (Iterable[tuple[str, int, str]]): (instance token, time, label token) of each + 3D box (at its sample's time) and 2D label (at its frame's time). + + Returns: + Count, first and last label token, keyed by instance token. + """ + times: dict[str, list[tuple[int, str]]] = defaultdict(list) + for instance_token, timestamp, token in labels: + times[instance_token].append((timestamp, token)) + return { + token: (len(entries), min(entries)[1], max(entries)[1]) + for token, entries in times.items() + } diff --git a/t4_devkit/schema/v1/keypoint.py b/t4_devkit/schema/v1/keypoint.py new file mode 100644 index 00000000..e18c4916 --- /dev/null +++ b/t4_devkit/schema/v1/keypoint.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import numpy as np +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .annotation_type import AnnotationType +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Keypoint"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.KEYPOINT) +class Keypoint(SchemaBase): + """`keypoint.json` dataclass. + + Attributes: + token (str): Unique record identifier. + sample_data_token (str): Foreign key pointing to the sample data. + instance_token (str): Foreign key pointing to the instance. + category_tokens (list[str]): Foreign keys pointing to the category of each keypoint. + keypoints (np.ndarray): Keypoints in shape (N, 2), as (x, y) in pixels. + num_keypoints (int): Number of keypoints. + annotation_type (AnnotationType): How the label was made. + provenance (str | None): Vendor, model name and version, or tool. + confidence (float | None): Confidence in [0, 1], or None for manual labels. + """ + + sample_data_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + instance_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + category_tokens: list[str] = field( + validator=validators.deep_iterable( + validators.and_(validators.instance_of(str), impossible_empty()), + validators.instance_of(list), + ) + ) + keypoints: np.ndarray = field(converter=np.array) + num_keypoints: int = field(validator=(validators.instance_of(int), validators.ge(0))) + annotation_type: AnnotationType = field(converter=AnnotationType) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) + + @keypoints.validator + def _check_keypoints(self, attribute, value: np.ndarray) -> None: + if value.ndim != 2 or value.shape[1] != 2: + raise ValueError(f"keypoints must have shape (N, 2), got {value.shape}") diff --git a/t4_devkit/schema/v1/loader.py b/t4_devkit/schema/v1/loader.py new file mode 100644 index 00000000..647de3b9 --- /dev/null +++ b/t4_devkit/schema/v1/loader.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import os.path as osp +from typing import TYPE_CHECKING, Any + +from attrs import define, field + +from t4_devkit.common.io import load_json + +from .convert_v0 import V0Converter +from .manifest import Manifest, load_manifest +from .name import SchemaName +from .registry import SCHEMAS + +if TYPE_CHECKING: + from t4_devkit.schema.tables.base import SchemaTable + from t4_devkit.typing import PathLike + +__all__ = ["T4Tables", "load_dataset"] + + +@define +class T4Tables: + """The tables of a loaded dataset, as 1.x records. + + Attributes: + data_root (str): Dataset root. + manifest (Manifest | None): The dataset manifest, or None for a 0.x dataset. + tables (dict[SchemaName, list[SchemaTable]]): Records per table; empty for an absent optional table. + """ + + data_root: str + manifest: Manifest | None + tables: dict[SchemaName, list[SchemaTable]] + _token2idx: dict[SchemaName, dict[str, int]] = field(init=False) + + def __attrs_post_init__(self) -> None: + self._token2idx = { + name: {record.token: idx for idx, record in enumerate(records)} + for name, records in self.tables.items() + } + self._fill_shortcuts() + + def get(self, name: str | SchemaName, token: str) -> SchemaTable: + """Return the record of a table by its token. + + Args: + name (str | SchemaName): Table name. + token (str): Record token. + + Returns: + The record. + """ + name = SchemaName(name) + return self.tables[name][self.get_idx(name, token)] + + def get_idx(self, name: str | SchemaName, token: str) -> int: + """Return the position of a record in its table by its token. + + Args: + name (str | SchemaName): Table name. + token (str): Record token. + + Returns: + The position in `tables[name]`. + + Raises: + KeyError: If no record of the table has this token. + """ + name = SchemaName(name) + idx = self._token2idx[name].get(token) + if idx is None: + raise KeyError(f"{token} is not registered in {name.value}.") + return idx + + def _fill_shortcuts(self) -> None: + """Fill the shortcut fields (`init=False`) of the records, from their links.""" + for record in self.tables[SchemaName.SAMPLE_ANNOTATION]: + instance = self.get(SchemaName.INSTANCE, record.instance_token) + record.category_name = self.get(SchemaName.CATEGORY, instance.category_token).name + + for record in self.tables[SchemaName.OBJECT_ANN]: + record.category_name = self.get(SchemaName.CATEGORY, record.category_token).name + + for record in self.tables[SchemaName.SAMPLE_DATA]: + sensor = self.get(SchemaName.SENSOR, record.sensor_token) + record.modality = sensor.modality + record.channel = sensor.channel + first = sensor.first_sd_token + if ( + first is None + or record.timestamp < self.get(SchemaName.SAMPLE_DATA, first).timestamp + ): + sensor.first_sd_token = record.token + # every frame belongs to its sample, key frame or not + self.get(SchemaName.SAMPLE, record.sample_token).data[sensor.channel] = record.token + + for record in self.tables[SchemaName.SAMPLE_ANNOTATION]: + self.get(SchemaName.SAMPLE, record.sample_token).ann_3ds.append(record.token) + + for record in self.tables[SchemaName.OBJECT_ANN]: + sample_data = self.get(SchemaName.SAMPLE_DATA, record.sample_data_token) + self.get(SchemaName.SAMPLE, sample_data.sample_token).ann_2ds.append(record.token) + + +def load_dataset(data_root: PathLike) -> T4Tables: + """Load a dataset as 1.x records. + + Read the raw table dicts (0.x: converted to 1.0 dicts, best effort) -> build and validate + the records once. + + Args: + data_root (PathLike): Dataset root, i.e. the `/` directory. + + Returns: + The loaded tables. + + Raises: + FileNotFoundError: If `manifest.json` does not list a required table. + """ + manifest = load_manifest(data_root) + raw_tables = ( + V0Converter(data_root).convert() if manifest is None else _read_tables(data_root, manifest) + ) + tables = { + name: [SCHEMAS.get(name).from_dict(data) for data in records] + for name, records in raw_tables.items() + } + return T4Tables(data_root=str(data_root), manifest=manifest, tables=tables) + + +def _read_tables(data_root: PathLike, manifest: Manifest) -> dict[SchemaName, list[dict[str, Any]]]: + """Read the raw table dicts listed in `manifest.json`. + + Args: + data_root (PathLike): Dataset root. + manifest (Manifest): The dataset manifest. + + Returns: + Raw records per table; empty for an absent optional table. + + Raises: + FileNotFoundError: If `manifest.json` does not list a required table. + """ + raw_tables: dict[SchemaName, list[dict[str, Any]]] = {} + for name in SchemaName: + entry = manifest.tables.get(name.value) + if entry is None: + if not name.is_optional(): + raise FileNotFoundError(f"{name.value} is mandatory.") + raw_tables[name] = [] + else: + raw_tables[name] = load_json(osp.join(data_root, entry.path)) + return raw_tables diff --git a/t4_devkit/schema/v1/log.py b/t4_devkit/schema/v1/log.py new file mode 100644 index 00000000..3440854a --- /dev/null +++ b/t4_devkit/schema/v1/log.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Log"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.LOG) +class Log(SchemaBase): + """`log.json` dataclass. + + Attributes: + token (str): Unique record identifier. + logfile (str | None): Log file name. + vehicle_token (str): Foreign key pointing to the vehicle. + data_captured (str): Capture time, ISO 8601. + location (str | None): Location of the recording. + map_token (str | None): Foreign key pointing to the map, or None if there is no map. + """ + + logfile: str | None = field(validator=validators.optional(validators.instance_of(str))) + vehicle_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + data_captured: str = field(validator=validators.instance_of(str)) + location: str | None = field(validator=validators.optional(validators.instance_of(str))) + map_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) diff --git a/t4_devkit/schema/v1/manifest.py b/t4_devkit/schema/v1/manifest.py new file mode 100644 index 00000000..755d9280 --- /dev/null +++ b/t4_devkit/schema/v1/manifest.py @@ -0,0 +1,406 @@ +from __future__ import annotations + +import os.path as osp +from enum import Enum, unique +from typing import TYPE_CHECKING, Any, ClassVar, Mapping, Sequence + +from attrs import converters, define, field, validators + +from t4_devkit.common.io import load_json +from t4_devkit.schema.tables.sensor import SensorModality + +from .name import SchemaName + +if TYPE_CHECKING: + from t4_devkit.typing import PathLike + +__all__ = [ + "MANIFEST_FILENAME", + "SUPPORTED_SCHEMA_MAJOR_VERSION", + "Task", + "RosbagStorage", + "Generator", + "Modalities", + "Capabilities", + "AnnotationSpecification", + "TableEntry", + "AssetEntry", + "Rosbag", + "Provenance", + "Manifest", + "load_manifest", +] + +MANIFEST_FILENAME = "manifest.json" +SUPPORTED_SCHEMA_MAJOR_VERSION = 1 + + +@unique +class Task(str, Enum): + """Tasks a dataset can be used for.""" + + DETECTION3D = "detection3d" + DETECTION2D = "detection2d" + SEGMENTATION2D = "segmentation2d" + SEGMENTATION3D = "segmentation3d" + KEYPOINT = "keypoint" + TRAFFIC_LIGHT = "traffic_light" + PREDICTION = "prediction" + PLANNING = "planning" + + +@unique +class RosbagStorage(str, Enum): + """Storage format of the source rosbag.""" + + SQLITE3 = "sqlite3" + MCAP = "mcap" + OTHER = "other" + + +@define +class Generator: + """A tool or library that produced the dataset. + + Attributes: + name (str): Tool or library name. + version (str): Version or commit hash. + """ + + name: str = field(validator=validators.instance_of(str)) + version: str = field(validator=validators.instance_of(str)) + + +@define +class Modalities: + """Sensor channel names per modality. + + Attributes: + camera (list[str]): Camera channel names. + lidar (list[str]): LiDAR channel names. + radar (list[str]): Radar channel names. + """ + + camera: list[str] = field( + validator=validators.deep_iterable( + validators.instance_of(str), validators.instance_of(list) + ) + ) + lidar: list[str] = field( + validator=validators.deep_iterable( + validators.instance_of(str), validators.instance_of(list) + ) + ) + radar: list[str] = field( + validator=validators.deep_iterable( + validators.instance_of(str), validators.instance_of(list) + ) + ) + + +@define +class Capabilities: + """What the dataset contains. + + Attributes: + tasks (list[Task]): Tasks the dataset can be used for. + modalities (Modalities): Sensor channel names per modality. + map (bool): Whether a lanelet2 map is included. + route (bool): Whether the `route` table is filled. + traffic_light (bool): Whether the `traffic_light` table is filled. + vehicle_state (bool): Whether the `vehicle_state` table is filled. + scenario (bool): Whether the `scenario` table is filled. + """ + + tasks: list[Task] = field(converter=lambda values: [Task(v) for v in values]) + modalities: Modalities = field( + converter=lambda x: Modalities(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(Modalities), + ) + map: bool = field(validator=validators.instance_of(bool)) + route: bool = field(validator=validators.instance_of(bool)) + traffic_light: bool = field(validator=validators.instance_of(bool)) + vehicle_state: bool = field(validator=validators.instance_of(bool)) + scenario: bool = field(validator=validators.instance_of(bool)) + + @classmethod + def from_tables(cls, tables: Mapping[SchemaName, Sequence[Any]]) -> Capabilities: + """Return what the tables contain: channels with frames, filled tables, tasks with labels. + + `prediction` needs every sample to have a key frame (all frames annotated); `planning` + needs scenarios. + + Args: + tables (Mapping[SchemaName, Sequence[Any]]): 1.0 records per table; a missing table + is empty. + + Returns: + The capabilities. + """ + + def records(table: SchemaName) -> Sequence[Any]: + return tables.get(table, []) + + sensors = records(SchemaName.SENSOR) + with_frames = {frame.sensor_token for frame in records(SchemaName.SAMPLE_DATA)} + channels = { + modality: list( + dict.fromkeys( + sensor.channel + for sensor in sensors + if sensor.modality == modality and sensor.token in with_frames + ) + ) + for modality in SensorModality + } + lidar_columns = { + field.name + for sensor in sensors + if sensor.lidar is not None + for field in sensor.lidar.fields + } + samples = records(SchemaName.SAMPLE) + things = {category.token for category in records(SchemaName.CATEGORY) if category.is_thing} + object_anns = records(SchemaName.OBJECT_ANN) + has_task = { + Task.DETECTION3D: bool(records(SchemaName.SAMPLE_ANNOTATION)), + Task.DETECTION2D: any(ann.category_token in things for ann in object_anns), + Task.SEGMENTATION2D: any(ann.mask is not None for ann in object_anns), + Task.SEGMENTATION3D: "label" in lidar_columns, + Task.KEYPOINT: bool(records(SchemaName.KEYPOINT)), + # states, or lamp instances linked to the map + Task.TRAFFIC_LIGHT: bool(records(SchemaName.TRAFFIC_LIGHT)) + or any( + instance.map_element.id is not None for instance in records(SchemaName.INSTANCE) + ), + Task.PREDICTION: bool(samples) and all(sample.has_key_frame for sample in samples), + Task.PLANNING: bool(records(SchemaName.SCENARIO)), + } + return cls( + tasks=[task for task in Task if has_task[task]], + modalities=Modalities( + camera=channels[SensorModality.CAMERA], + lidar=channels[SensorModality.LIDAR], + radar=channels[SensorModality.RADAR], + ), + map=bool(records(SchemaName.MAP)), + route=bool(records(SchemaName.ROUTE)), + traffic_light=bool(records(SchemaName.TRAFFIC_LIGHT)), + vehicle_state=bool(records(SchemaName.VEHICLE_STATE)), + scenario=bool(records(SchemaName.SCENARIO)), + ) + + +@define +class AnnotationSpecification: + """Annotation specification of a task. + + Attributes: + id (str): Annotation specification ID. + version (str | None): Specification version, or None if unknown. + """ + + id: str = field(validator=validators.instance_of(str)) + version: str | None = field(validator=validators.optional(validators.instance_of(str))) + + +@define +class TableEntry: + """A table file and its hash. + + Attributes: + path (str): Path relative to the dataset root. + sha256 (str): SHA-256 of the file. + """ + + path: str = field(validator=validators.instance_of(str)) + sha256: str = field(validator=validators.matches_re(r"^[0-9a-f]{64}$")) + + +@define +class AssetEntry: + """A file asset, as in sensor, map or input bag file with its hash and size. + + Attributes: + path (str): Path relative to the dataset root. + sha256 (str): SHA-256 of the file. + bytes (int): File size in bytes. + """ + + path: str = field(validator=validators.instance_of(str)) + sha256: str = field(validator=validators.matches_re(r"^[0-9a-f]{64}$")) + bytes: int = field(validator=(validators.instance_of(int), validators.ge(0))) + + +@define +class Rosbag: + """Source rosbags of the dataset as defined by MOB. + + Attributes: + log_file_ids (list[str]): Web.Auto log file IDs. + storage (RosbagStorage | None): rosbag storage format. + record_start (str): Recording start, ISO 8601 UTC. + record_end (str): Recording end, ISO 8601 UTC. + """ + + log_file_ids: list[str] = field( + validator=validators.deep_iterable( + validators.instance_of(str), validators.instance_of(list) + ) + ) + storage: RosbagStorage | None = field(converter=converters.optional(RosbagStorage)) + record_start: str = field(validator=validators.instance_of(str)) + record_end: str = field(validator=validators.instance_of(str)) + + +@define +class Provenance: + """Where this t4dataset came from. + + Attributes: + rosbag (Rosbag): Source rosbag. + topics (dict[str, str]): ROS topic used for each channel. + process_config_id (str | None): Web.Auto processing configuration ID. + process_config_version_id (int | None): Web.Auto processing configuration version. + """ + + rosbag: Rosbag = field( + converter=lambda x: Rosbag(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(Rosbag), + ) + topics: dict[str, str] = field( + validator=validators.deep_mapping( + validators.instance_of(str), validators.instance_of(str), validators.instance_of(dict) + ) + ) + process_config_id: str | None = field( + validator=validators.optional(validators.instance_of(str)) + ) + process_config_version_id: int | None = field( + validator=validators.optional(validators.instance_of(int)) + ) + + +@define +class Manifest: + """`manifest.json` dataclass. + + Attributes: + schema_version (str): Version of the dataset specification, e.g. "1.0.0". + dataset_id (str): Dataset ID. + version (int): Dataset version. + created_at (str): Creation time, ISO 8601. + generators (list[Generator]): Tools and libraries that produced the dataset. + capabilities (Capabilities): What the dataset contains. + annotation_specifications (dict[str, AnnotationSpecification]): Specification per task. + tables (dict[str, TableEntry]): Table files, keyed by table name. + assets (list[AssetEntry]): Sensor, map and input bag files. + provenance (Provenance): Where the data came from. + project_id (str | None): Web.Auto project ID. + source_project_id (str | None): Web.Auto source project ID. + is_synthetic (bool): Whether the data is synthetic. + is_discontinuous (bool | None): Whether the recording has gaps. + migrated_from_schema_version (str | None): Source version if the dataset was migrated. + migration_notes (list[str] | None): Notes on anything lost or changed during migration. + """ + + # Folders whose files are listed in `assets` (the tables, under `annotation/`, in `tables`). + ASSET_DIRS: ClassVar[tuple[str, ...]] = ("data", "map", "input_bag") + + schema_version: str = field(validator=validators.instance_of(str)) + dataset_id: str = field(validator=validators.instance_of(str)) + version: int = field(validator=(validators.instance_of(int), validators.ge(0))) + created_at: str = field(validator=validators.instance_of(str)) + generators: list[Generator] = field( + converter=lambda x: [Generator(**v) if isinstance(v, dict) else v for v in x], + validator=validators.deep_iterable( + validators.instance_of(Generator), validators.instance_of(list) + ), + ) + capabilities: Capabilities = field( + converter=lambda x: Capabilities(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(Capabilities), + ) + annotation_specifications: dict[str, AnnotationSpecification] = field( + converter=lambda x: { + k: AnnotationSpecification(**v) if isinstance(v, dict) else v for k, v in x.items() + }, + validator=validators.deep_mapping( + validators.instance_of(str), + validators.instance_of(AnnotationSpecification), + validators.instance_of(dict), + ), + ) + tables: dict[str, TableEntry] = field( + converter=lambda x: { + k: TableEntry(**v) if isinstance(v, dict) else v for k, v in x.items() + }, + validator=validators.deep_mapping( + validators.instance_of(str), + validators.instance_of(TableEntry), + validators.instance_of(dict), + ), + ) + assets: list[AssetEntry] = field( + converter=lambda x: [AssetEntry(**v) if isinstance(v, dict) else v for v in x], + validator=validators.deep_iterable( + validators.instance_of(AssetEntry), validators.instance_of(list) + ), + ) + provenance: Provenance = field( + converter=lambda x: Provenance(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(Provenance), + ) + project_id: str | None = field(validator=validators.optional(validators.instance_of(str))) + source_project_id: str | None = field( + validator=validators.optional(validators.instance_of(str)) + ) + is_synthetic: bool = field(validator=validators.instance_of(bool)) + is_discontinuous: bool | None = field( + validator=validators.optional(validators.instance_of(bool)) + ) + migrated_from_schema_version: str | None = field( + validator=validators.optional(validators.instance_of(str)) + ) + migration_notes: list[str] | None = field( + validator=validators.optional( + validators.deep_iterable(validators.instance_of(str), validators.instance_of(list)) + ) + ) + + def listed_paths(self) -> list[str]: + """Return the path of every file listed: tables, then assets. + + Returns: + Paths relative to the dataset root. + """ + return [entry.path for entry in self.tables.values()] + [ + entry.path for entry in self.assets + ] + + @schema_version.validator + def _check_schema_version(self, attribute, value: str) -> None: + major = value.split(".")[0] + if major != str(SUPPORTED_SCHEMA_MAJOR_VERSION): + raise ValueError( + f"Unsupported schema_version {value!r}: " + f"this t4-devkit reads {SUPPORTED_SCHEMA_MAJOR_VERSION}.x." + ) + + +def load_manifest(data_root: PathLike) -> Manifest | None: + """Load `manifest.json` from a dataset root. + + Args: + data_root (PathLike): Dataset root, i.e. the `/` directory. + + Returns: + The manifest of a T4Dataset 1.0 or later, or None for a 0.x dataset (no `manifest.json`). + + Raises: + ValueError: If the manifest declares a schema version this t4-devkit does not support. + """ + filepath = osp.join(data_root, MANIFEST_FILENAME) + if not osp.exists(filepath): + return None + return Manifest(**load_json(filepath)) diff --git a/t4_devkit/schema/v1/map.py b/t4_devkit/schema/v1/map.py new file mode 100644 index 00000000..bd050a23 --- /dev/null +++ b/t4_devkit/schema/v1/map.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import converters, define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = [ + "MapFormat", + "ProjectorType", + "MapSource", + "ProjectorOrigin", + "Projector", + "Map", + "MapElementType", + "MapElement", +] + + +@unique +class MapFormat(str, Enum): + """File format of a map.""" + + LANELET2_OSM = "lanelet2_osm" + + +@unique +class ProjectorType(str, Enum): + """Map projection, as in Autoware's map_projector_info.""" + + MGRS = "MGRS" + TRANSVERSE_MERCATOR = "TransverseMercator" + LOCAL_CARTESIAN_UTM = "LocalCartesianUTM" + + +@unique +class MapSource(str, Enum): + """Where a map came from.""" + + WEBAUTO = "webauto" + MANUAL = "manual" + + +@define +class ProjectorOrigin: + """Origin of the map projection. + + Attributes: + lat (float | None): Latitude [deg]. + lon (float | None): Longitude [deg]. + alt (float | None): Altitude [m]. + """ + + lat: float | None = field(validator=validators.optional(validators.instance_of((int, float)))) + lon: float | None = field(validator=validators.optional(validators.instance_of((int, float)))) + alt: float | None = field(validator=validators.optional(validators.instance_of((int, float)))) + + +@define +class Projector: + """Map projection. + + Attributes: + type (ProjectorType | None): Projection type. + mgrs_grid (str | None): MGRS grid. + origin (ProjectorOrigin): Projection origin. + """ + + type: ProjectorType | None = field(converter=converters.optional(ProjectorType)) + mgrs_grid: str | None = field(validator=validators.optional(validators.instance_of(str))) + origin: ProjectorOrigin = field( + converter=lambda x: ProjectorOrigin(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(ProjectorOrigin), + ) + + +@unique +class MapElementType(str, Enum): + """lanelet2 element type. Element IDs are only unique within a type.""" + + NODE = "node" + WAY = "way" + RELATION = "relation" + + +@define +class MapElement: + """A reference to a lanelet2 element, e.g. from an instance. + + Attributes: + id (int | None): lanelet2 element ID: traffic-light lamp, sign, crosswalk, stop line, lane. + type (MapElementType | None): lanelet2 element type. + """ + + id: int | None = field(validator=validators.optional(validators.instance_of(int))) + type: MapElementType | None = field(converter=converters.optional(MapElementType)) + + +@define(slots=False) +@SCHEMAS.register(SchemaName.MAP) +class Map(SchemaBase): + """`map.json` dataclass. + + Lane data stays in the lanelet2 file; this table holds the map identity. + + Attributes: + token (str): Unique record identifier. + name (str | None): Map name. + log_tokens (list[str]): Foreign keys pointing to the logs. + format (MapFormat): Map file format. + filename (str): Path to the map file. + sha256 (str): SHA-256 of the map file. + map_id (str): Web.Auto area map ID. + version_id (str): Web.Auto area map version ID. + projector (Projector): Map projection. + source (MapSource): Where the map came from. + """ + + name: str | None = field(validator=validators.optional(validators.instance_of(str))) + log_tokens: list[str] = field( + validator=validators.deep_iterable( + (validators.instance_of(str), impossible_empty()), validators.instance_of(list) + ) + ) + format: MapFormat = field(converter=MapFormat) + filename: str = field(validator=(validators.instance_of(str), impossible_empty())) + sha256: str = field(validator=validators.matches_re(r"^[0-9a-f]{64}$")) + map_id: str = field(validator=(validators.instance_of(str), impossible_empty())) + version_id: str = field(validator=(validators.instance_of(str), impossible_empty())) + projector: Projector = field( + converter=lambda x: Projector(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(Projector), + ) + source: MapSource = field(converter=MapSource) diff --git a/t4_devkit/schema/v1/migrate_v0.py b/t4_devkit/schema/v1/migrate_v0.py new file mode 100644 index 00000000..560d1fef --- /dev/null +++ b/t4_devkit/schema/v1/migrate_v0.py @@ -0,0 +1,448 @@ +from __future__ import annotations + +import os.path as osp +import warnings +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Mapping + +import numpy as np + +from t4_devkit.common.io import load_image, load_json, load_pointcloud + +from .loader import load_dataset +from .name import SchemaName +from .sensor import LidarContainer, LidarField +from .writer import DatasetWriter + +if TYPE_CHECKING: + from t4_devkit.typing import PathLike + + from .manifest import Manifest + from .sample_data import SampleData + from .sensor import Sensor + +__all__ = ["V0Migrator"] + + +class V0Migrator: + """Migrate a 0.x dataset to a new 1.0 dataset folder; the 0.x dataset is only read. + + Tables are loaded as 1.0 records (`load_dataset`), then each channel's files are rewritten + in the 1.0 layout and its `sensor` / `sample_data` records updated to point to them. What + the migration had to decide is collected in `notes`, for `manifest.migration_notes`. + + Examples: + >>> migrator = V0Migrator("<0.x dataset>", "out//") + >>> manifest = migrator.migrate("", ) + + Or channel by channel: + + >>> for sensor in migrator.lidars(): + ... migrator.migrate_lidar(sensor) + >>> for sensor in migrator.cameras(): + ... migrator.migrate_camera(sensor) # or copy_images(sensor) to keep the files + """ + + def __init__(self, data_root: PathLike, output_root: PathLike) -> None: + """ + Args: + data_root (PathLike): 0.x dataset root, i.e. `/`. + output_root (PathLike): 1.0 dataset root to create. + """ + self.data_root = data_root + self.dataset = load_dataset(data_root) + self.writer = DatasetWriter(output_root) + self.notes: list[str] = [] + self._v0_frames = {record["token"]: record for record in self._load_v0("sample_data")} + self._v0_channels = { + record["token"]: record["channel"] for record in self._load_v0("sensor") + } + self._sensor_indices = { + sensor.channel: sensor.index for sensor in self.tables(SchemaName.SENSOR) + } + self._labels = { + record["sample_data_token"]: record["filename"] for record in self._load_v0("lidarseg") + } + + def lidars(self) -> list[Sensor]: + """Return the LiDAR sensors with frames of their own.""" + with_frames = {frame.sensor_token for frame in self.tables(SchemaName.SAMPLE_DATA)} + return [ + sensor + for sensor in self.tables(SchemaName.SENSOR) + if sensor.lidar is not None and sensor.token in with_frames + ] + + def migrate_lidar(self, sensor: Sensor) -> None: + """Write the frames of a LiDAR channel to one Parquet file and point its records to it. + + Point fields get their 1.0 dtypes; `source_index` / `source_timestamp` come from the + 0.x `.info` files, and `label` / `instance` from 0.x lidarseg (`instance` is -1: 0.x + labels are semantic only; frames without labels are null). + + Args: + sensor (Sensor): LiDAR sensor with frames. + + Raises: + ValueError: If the points of a frame are not covered by its `.info` sources, or + lidarseg is present but category 0 is not `unpainted`. + """ + records = self._channel_frames(sensor) + self._relink(sensor, records) + # the 0.x layout, as loading read it (`.info` field count, or the `.pcd` header) + source_fields = [(field.name, field.dtype) for field in sensor.lidar.fields] + fields = self._point_fields(sensor.channel, records, source_fields) + if self._v0_frames[records[0].token].get("info_filename"): + fields += [("source_index", "uint8"), ("source_timestamp", "int64")] + if any(record.token in self._labels for record in records): + self._check_unpainted() + self._index_instances() + fields += [("label", "uint8"), ("instance", "int16")] + + frames = (self._lidar_frame(record, source_fields, fields) for record in records) + path, frame_count = self.writer.write_lidar(sensor.channel, frames, fields) + + sensor.lidar.container = LidarContainer( + format="parquet", path=path, frame_count=frame_count, compression="zstd" + ) + sensor.lidar.fields = [LidarField(name=name, dtype=dtype) for name, dtype in fields] + for frame_index, record in enumerate(records): + record.filename = path + record.frame_index = frame_index + record.fileformat = "parquet" + + # Source LiDARs of a merged channel have no frames of their own: their points are in + # this file, so they get its format and fields. + with_frames = {lidar.token for lidar in self.lidars()} + for source in self.tables(SchemaName.SENSOR): + if source.channel in sensor.lidar.source_channels and source.token not in with_frames: + source.lidar.container = LidarContainer( + format="parquet", + path=None, + frame_count=source.lidar.container.frame_count, + compression="zstd", + ) + source.lidar.fields = list(sensor.lidar.fields) + + def cameras(self) -> list[Sensor]: + """Return the camera sensors with frames.""" + with_frames = {frame.sensor_token for frame in self.tables(SchemaName.SAMPLE_DATA)} + return [ + sensor + for sensor in self.tables(SchemaName.SENSOR) + if sensor.camera is not None and sensor.token in with_frames + ] + + def migrate_camera(self, sensor: Sensor) -> None: + """Encode the image frames of a camera channel as one AV1 video and point its records to it. + + Frames go in timestamp order, so `frame_index` is the position. Frames marked + `is_valid: false` (blank images) are encoded like the others. The frame rate is the + median interval between frames. + + Args: + sensor (Sensor): Camera sensor with frames. + + Raises: + ValueError: If an image does not have the sensor's width and height. + """ + records = self._channel_frames(sensor) + self._relink(sensor, records) + images = (self._image(record, sensor) for record in records) + video = self.writer.write_camera(sensor.channel, images, self._frame_rate(records)) + + sensor.camera.video = video + for frame_index, record in enumerate(records): + record.filename = video["path"] + record.frame_index = frame_index + record.fileformat = "av1_mp4" + + def copy_images(self, sensor: Sensor) -> None: + """Copy the image files of a camera channel as they are, instead of encoding a video. + + Args: + sensor (Sensor): Camera sensor with frames. + """ + records = self._channel_frames(sensor) + self._relink(sensor, records) + for record in records: + self.writer.copy(osp.join(self.data_root, record.filename), record.filename) + + def migrate( + self, + dataset_id: str, + version: int, + *, + keep_images: bool = False, + manifest_fields: Mapping[str, Any] | None = None, + ) -> Manifest: + """Migrate every channel, copy `map/` and `input_bag/`, then write the tables and + `manifest.json`. + + What 0.x does not store can be given by the caller: records through `tables`, and + manifest fields (e.g. `provenance`, `project_id`, `generators`) through + `manifest_fields`. + + Args: + dataset_id (str): Dataset ID, as in the output folder. + version (int): Dataset version, as in the output folder (given by Web.Auto). + keep_images (bool, optional): Copy camera images as they are instead of encoding + one video per channel. + manifest_fields (Mapping[str, Any] | None, optional): Manifest fields that replace + the defaults; `migration_notes` are added after the migration's own. + + Returns: + The manifest written. + """ + for sensor in self.lidars(): + self.migrate_lidar(sensor) + for sensor in self.cameras(): + if keep_images: + self.copy_images(sensor) + else: + self.migrate_camera(sensor) + for directory in ("map", "input_bag"): + if osp.isdir(osp.join(self.data_root, directory)): + self.writer.copy(osp.join(self.data_root, directory), directory) + fields = dict(manifest_fields or {}) + if "provenance" not in fields: + self.notes.append( + "provenance.rosbag.record_start / record_end are the first / last sample times" + ) + if any(record.get("acceleration") for record in self._load_v0("ego_pose")): + self.notes.append( + "ego_pose.acceleration dropped: 0.x stored raw IMU readings, whose frame and " + "gravity sign vary by vehicle; the backfill restores acceleration and imu" + ) + manifest_info = self._manifest_info(dataset_id, version) + manifest_info.update(fields) + manifest_info["migration_notes"] = self.notes + list(fields.get("migration_notes", [])) + return self.writer.save(self.dataset.tables, manifest_info) + + def tables(self, table: SchemaName) -> list[Any]: + """Return the 1.0 records of a table.""" + return self.dataset.tables[table] + + def _manifest_info(self, dataset_id: str, version: int) -> dict[str, Any]: + """Return the default manifest fields the writer does not compute. 0.x has no + provenance, so it is left empty for backfilling, except the recording time (from the + samples).""" + timestamps = [sample.timestamp for sample in self.tables(SchemaName.SAMPLE)] + + def utc(timestamp: int) -> str: + return datetime.fromtimestamp(timestamp / 1e6, tz=timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + + return { + "schema_version": "1.0.0", + "dataset_id": dataset_id, + "version": version, + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "generators": [], + "annotation_specifications": {}, + "provenance": { + "rosbag": { + "log_file_ids": [], + "storage": None, + "record_start": utc(min(timestamps)), + "record_end": utc(max(timestamps)), + }, + "topics": {}, + "process_config_id": None, + "process_config_version_id": None, + }, + "project_id": None, + "source_project_id": None, + "is_synthetic": False, + "is_discontinuous": None, + "migrated_from_schema_version": "0.x", + } + + def _channel_frames(self, sensor: Sensor) -> list[SampleData]: + """Return the frames of a sensor in timestamp order.""" + return sorted( + ( + frame + for frame in self.tables(SchemaName.SAMPLE_DATA) + if frame.sensor_token == sensor.token + ), + key=lambda frame: frame.timestamp, + ) + + def _relink(self, sensor: Sensor, records: list[SampleData]) -> None: + """Set `next` / `prev` of a channel's frames from their timestamp order. + + Some 0.x datasets leave a placeholder frame (`is_valid: false`) half linked: it points + to its neighbors, but they point to each other. Each change is noted. + """ + relinked = 0 + for i, record in enumerate(records): + prev = records[i - 1].token if i > 0 else None + next_ = records[i + 1].token if i + 1 < len(records) else None + if (record.prev, record.next) != (prev, next_): + record.prev, record.next = prev, next_ + relinked += 1 + if relinked: + self.notes.append( + f"{sensor.channel}: next / prev relinked in time order on {relinked} frames" + ) + + @staticmethod + def _frame_rate(records: list[SampleData]) -> int: + """Return the frame rate of a channel: the median interval between frames, in Hz.""" + if len(records) < 2: + return 10 # one frame: any rate plays it + interval = np.median(np.diff([record.timestamp for record in records])) + return max(1, round(1_000_000 / interval)) + + def _image(self, record: SampleData, sensor: Sensor) -> np.ndarray: + image = load_image(osp.join(self.data_root, record.filename)) + if image.shape[:2] != (sensor.camera.height, sensor.camera.width): + raise ValueError( + f"{record.filename}: {image.shape[1]}x{image.shape[0]}, but {sensor.channel} is " + f"{sensor.camera.width}x{sensor.camera.height}" + ) + return image + + def _point_fields( + self, channel: str, records: list[SampleData], source_fields: list[tuple[str, str]] + ) -> list[tuple[str, str]]: + """Return the 1.0 point fields of a channel, from the values of all its frames. + + A field gets its 1.0 dtype when every value fits it, and keeps its 0.x dtype otherwise + (with a warning). The ring (`channel`) is dropped when it is -1 on every point. + """ + # 1.0 dtypes of the 0.x point fields (Autoware layout); other fields keep their dtype + dtypes = { + "x": "float32", + "y": "float32", + "z": "float32", + "intensity": "uint8", + "channel": "uint16", + "return_type": "uint8", + "time_stamp": "uint32", + } + + # Over every frame: each field's smallest and largest value, and whether all its + # values are whole numbers. + low = {name: np.inf for name, _ in source_fields} + high = {name: -np.inf for name, _ in source_fields} + whole = {name: True for name, _ in source_fields} + for record in records: + for name, values in self._read_points(record, source_fields).items(): + if len(values) == 0: + continue + low[name] = min(low[name], values.min()) + high[name] = max(high[name], values.max()) + whole[name] = whole[name] and np.array_equal(values, np.round(values)) + + # Decide each field. + fields = [] + for name, dtype in source_fields: + target = np.dtype(dtypes.get(name, dtype)) + + # Ring not recorded: the 0.x converter writes -1 on every point. + if name == "channel" and low[name] == high[name] == -1: + self.notes.append(f"{channel}: '{name}' dropped, -1 on every point (not recorded)") + continue + + # Integer target: values must be whole and in range. Float target: always fits. + no_points = low[name] > high[name] + fits = ( + not np.issubdtype(target, np.integer) + or no_points + or ( + whole[name] + and np.iinfo(target).min <= low[name] + and high[name] <= np.iinfo(target).max + ) + ) + if fits: + fields.append((name, target.name)) + continue + + # Keep the 0.x dtype (lossless) if it doesn't fit, warn. + note = ( + f"{channel}: '{name}' kept as {dtype}, values in [{low[name]}, {high[name]}] " + f"do not fit {target.name}" + ) + warnings.warn(note) + self.notes.append(note) + fields.append((name, dtype)) + return fields + + def _lidar_frame( + self, + record: SampleData, + source_fields: list[tuple[str, str]], + fields: list[tuple[str, str]], + ) -> dict[str, np.ndarray | None]: + """Return one frame in the 1.0 layout, and set its `num_points`.""" + points = self._read_points(record, source_fields) + frame = {name: points[name].astype(dtype) for name, dtype in fields if name in points} + num_points = len(points[source_fields[0][0]]) + record.num_points = num_points + + info_filename = self._v0_frames[record.token].get("info_filename") + if info_filename: + frame |= self._sources(load_json(osp.join(self.data_root, info_filename)), num_points) + if ("label", "uint8") in fields: + labels = self._labels.get(record.token) + if labels is None: + frame["label"] = frame["instance"] = None + else: + label = np.fromfile(osp.join(self.data_root, labels), dtype=np.uint8) + if len(label) != num_points: + raise ValueError(f"{labels}: {len(label)} labels for {num_points} points") + frame["label"] = label + frame["instance"] = np.full(num_points, -1, dtype=np.int16) + return frame + + def _sources(self, info: dict, num_points: int) -> dict[str, np.ndarray]: + """Return `source_index` (the source's `sensor.index`) and `source_timestamp` (µs).""" + source_index = np.zeros(num_points, dtype=np.uint8) + source_timestamp = np.zeros(num_points, dtype=np.int64) + covered = np.zeros(num_points, dtype=bool) + for source in info.get("sources", []): + span = slice(source["idx_begin"], source["idx_begin"] + source["length"]) + source_index[span] = self._sensor_indices[self._v0_channels[source["sensor_token"]]] + stamp = source["stamp"] + source_timestamp[span] = stamp["sec"] * 1_000_000 + stamp["nanosec"] // 1000 + covered[span] = True + if not covered.all(): + raise ValueError(f"{(~covered).sum()} points are outside the .info sources") + return {"source_index": source_index, "source_timestamp": source_timestamp} + + def _read_points( + self, record: SampleData, source_fields: list[tuple[str, str]] + ) -> dict[str, np.ndarray]: + filepath = osp.join(self.data_root, record.filename) + if filepath.endswith(".pcd"): + return load_pointcloud(filepath) + return load_pointcloud(filepath, fields=source_fields) + + def _index_instances(self) -> None: + """Give instances without `index` one from 0, as point labels require. + + 0.x labels points by category only, so the `instance` column is -1 (no object), but + `REC011` still asks every instance of an `is_thing` category for an index. + """ + instances = self.tables(SchemaName.INSTANCE) + if all(instance.index is not None for instance in instances): + return + for index, instance in enumerate(instances): + instance.index = index + self.notes.append(f"instance.index set from 0 on {len(instances)} instances (0.x has none)") + + def _check_unpainted(self) -> None: + if not any( + category.index == 0 and category.name == "unpainted" + for category in self.tables(SchemaName.CATEGORY) + ): + raise ValueError("lidarseg is present, but category index 0 is not 'unpainted'") + + def _load_v0(self, table: str) -> list[dict]: + """Return the raw records of a 0.x table, or [] if it is absent.""" + filepath = osp.join(self.data_root, "annotation", f"{table}.json") + return load_json(filepath) if osp.exists(filepath) else [] diff --git a/t4_devkit/schema/v1/name.py b/t4_devkit/schema/v1/name.py new file mode 100644 index 00000000..91d4b34a --- /dev/null +++ b/t4_devkit/schema/v1/name.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from enum import Enum, unique + +__all__ = ["SchemaName"] + + +@unique +class SchemaName(str, Enum): + """Names of the T4Dataset 1.0 tables, one JSON file each in `annotation/`. + + Required tables: sample, sample_data, ego_pose, sensor, scene, log, vehicle. + All other tables are optional; `manifest.json` declares which ones a dataset contains. + """ + + SAMPLE = "sample" + SAMPLE_DATA = "sample_data" + EGO_POSE = "ego_pose" + SENSOR = "sensor" + SCENE = "scene" + LOG = "log" + VEHICLE = "vehicle" + MAP = "map" # optional + VEHICLE_STATE = "vehicle_state" # optional + CATEGORY = "category" # optional + ATTRIBUTE = "attribute" # optional + VISIBILITY = "visibility" # optional + INSTANCE = "instance" # optional + SAMPLE_ANNOTATION = "sample_annotation" # optional + OBJECT_ANN = "object_ann" # optional + KEYPOINT = "keypoint" # optional + TRAFFIC_LIGHT = "traffic_light" # optional + ROUTE = "route" # optional + SCENARIO = "scenario" # optional + CAPTION = "caption" # optional + + @property + def filename(self) -> str: + """Return the annotation json filename. + + Returns: + Annotation json filename. + """ + return f"{self.value}.json" + + def is_optional(self) -> bool: + """Indicates if this table is optional. + + Returns: + Return True if this table is optional. + """ + return self not in ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.EGO_POSE, + SchemaName.SENSOR, + SchemaName.SCENE, + SchemaName.LOG, + SchemaName.VEHICLE, + ) diff --git a/t4_devkit/schema/v1/object_ann.py b/t4_devkit/schema/v1/object_ann.py new file mode 100644 index 00000000..b648e3bd --- /dev/null +++ b/t4_devkit/schema/v1/object_ann.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty +from t4_devkit.schema.tables.object_ann import RLEMask + +from .annotation_type import AnnotationType +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["ObjectAnn"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.OBJECT_ANN) +class ObjectAnn(SchemaBase): + """`object_ann.json` dataclass. + + A 2D annotation in an image: a box with an optional mask. For categories with `is_thing`, + the mask is an instance mask and `instance_token` is set; for other categories (road, sky), + it is a semantic mask and `instance_token` is None. Replaces `object_ann` and `surface_ann` + of 0.x. + + Attributes: + token (str): Unique record identifier. + sample_data_token (str): Foreign key pointing to the sample data. + instance_token (str | None): Foreign key pointing to the instance, or None for categories + without `is_thing`. + category_token (str): Foreign key pointing to the category. + attribute_tokens (list[str]): Foreign keys pointing to the attributes. + bbox (tuple[float, float, float, float]): (xmin, ymin, xmax, ymax) in pixels. + mask (RLEMask | None): Instance mask, or semantic mask for categories without `is_thing`. + orientation (float | None): Arrow orientation [rad], for categories with `has_orientation`. + number (int | None): Displayed digit, for categories with `has_number`. + annotation_type (AnnotationType): How the label was made. + provenance (str | None): Vendor, model name and version, or tool. + confidence (float | None): Confidence in [0, 1], or None for manual labels. + + Shortcuts (filled by `T4Tables`): + --------- + category_name (str | None): Name of the category. + """ + + sample_data_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + instance_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + category_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + attribute_tokens: list[str] = field( + validator=validators.deep_iterable( + validators.and_(validators.instance_of(str), impossible_empty()), + validators.instance_of(list), + ) + ) + bbox: tuple[float, float, float, float] = field( + converter=tuple, + validator=validators.deep_iterable( + validators.instance_of((int, float)), + validators.and_(validators.min_len(4), validators.max_len(4)), + ), + ) + mask: RLEMask | None = field( + converter=lambda x: RLEMask(**x) if isinstance(x, dict) else x, + validator=validators.optional(validators.instance_of(RLEMask)), + ) + orientation: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + number: int | None = field(validator=validators.optional(validators.instance_of(int))) + annotation_type: AnnotationType = field(converter=AnnotationType) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) + + # shortcuts + category_name: str | None = field(init=False, default=None) diff --git a/t4_devkit/schema/v1/registry.py b/t4_devkit/schema/v1/registry.py new file mode 100644 index 00000000..4a4f48c0 --- /dev/null +++ b/t4_devkit/schema/v1/registry.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from t4_devkit.schema.tables.registry import SchemaRegistry + +from .name import SchemaName + +__all__ = ["SCHEMAS"] + +SCHEMAS = SchemaRegistry(SchemaName) diff --git a/t4_devkit/schema/v1/route.py b/t4_devkit/schema/v1/route.py new file mode 100644 index 00000000..aadc57b8 --- /dev/null +++ b/t4_devkit/schema/v1/route.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import define, field, validators + +from t4_devkit.common.converter import to_quaternion +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty +from t4_devkit.typing import Quaternion, Vector3 + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["RouteSource", "RoutePose", "RouteSegment", "Route"] + + +@unique +class RouteSource(str, Enum): + """Where a route came from.""" + + MISSION_PLANNER = "mission_planner" + MANUAL = "manual" + DERIVED = "derived" + + +@define +class RoutePose: + """A route start or goal pose in the map frame, as in LaneletRoute. + + Attributes: + translation (Vector3): Position [x, y, z] in [m]. + rotation (Quaternion): Orientation as quaternion [w, x, y, z]. + """ + + translation: Vector3 = field(converter=Vector3) + rotation: Quaternion = field(converter=to_quaternion) + + +@define +class RouteSegment: + """A route segment, as in Autoware LaneletSegment. + + Attributes: + preferred_primitive_id (int): Preferred lanelet ID. + primitive_ids (list[int]): All lanelet IDs of the segment. + """ + + preferred_primitive_id: int = field(validator=validators.instance_of(int)) + primitive_ids: list[int] = field( + validator=validators.deep_iterable( + validators.instance_of(int), validators.instance_of(list) + ) + ) + + +@define(slots=False) +@SCHEMAS.register(SchemaName.ROUTE) +class Route(SchemaBase): + """`route.json` dataclass. + + The mission route, from Autoware LaneletRoute. A re-route adds a new row. + + Attributes: + token (str): Unique record identifier. + log_token (str): Foreign key pointing to the log. + map_token (str): Foreign key pointing to the map. + valid_from (int): Start of validity, Unix time stamp [us] (inclusive). + valid_until (int | None): End of validity [us] (exclusive), or None until the end. + route_uuid (str | None): `LaneletRoute.uuid`. + start_pose (RoutePose): Start pose in the map frame. + goal_pose (RoutePose): Goal pose in the map frame. + segments (list[RouteSegment]): Route segments, in order. + allow_modification (bool | None): `LaneletRoute.allow_modification`. + source (RouteSource): Where the route came from. + provenance (str | None): Where the message was found, e.g. "own_bag", "route_json". + """ + + log_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + map_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + valid_from: int = field(validator=validators.instance_of(int)) + valid_until: int | None = field(validator=validators.optional(validators.instance_of(int))) + route_uuid: str | None = field(validator=validators.optional(validators.instance_of(str))) + start_pose: RoutePose = field( + converter=lambda x: RoutePose(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(RoutePose), + ) + goal_pose: RoutePose = field( + converter=lambda x: RoutePose(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(RoutePose), + ) + segments: list[RouteSegment] = field( + converter=lambda x: [RouteSegment(**v) if isinstance(v, dict) else v for v in x], + validator=validators.deep_iterable( + validators.instance_of(RouteSegment), validators.instance_of(list) + ), + ) + allow_modification: bool | None = field( + validator=validators.optional(validators.instance_of(bool)) + ) + source: RouteSource = field(converter=RouteSource) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) diff --git a/t4_devkit/schema/v1/sample.py b/t4_devkit/schema/v1/sample.py new file mode 100644 index 00000000..271006ef --- /dev/null +++ b/t4_devkit/schema/v1/sample.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Sample"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.SAMPLE) +class Sample(SchemaBase): + """`sample.json` dataclass. + + One frame of the recording: the observations of all channels at that moment. Every frame is a + sample, annotated or not. + + Attributes: + token (str): Unique record identifier. + timestamp (int): Unix time stamp [us] of the sample's LiDAR frame, or of its earliest frame + if it has no LiDAR. + scene_token (str): Foreign key pointing to the scene. + has_key_frame (bool): Whether any of its sample data is a key frame (annotated). + next (str | None): Foreign key pointing to the next sample, or None at the end of the scene. + prev (str | None): Foreign key pointing to the previous sample, or None at the start. + + Shortcuts (filled by `T4Tables`): + --------- + data (dict[str, str]): Sample data token per sensor channel (the channels of this frame). + ann_3ds (list[str]): Tokens of the sample annotations of this sample. + ann_2ds (list[str]): Tokens of the object annotations of this sample. + """ + + timestamp: int = field(validator=validators.instance_of(int)) + scene_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + has_key_frame: bool = field(validator=validators.instance_of(bool)) + next: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + prev: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + + # shortcuts + data: dict[str, str] = field(factory=dict, init=False) + ann_3ds: list[str] = field(factory=list, init=False) + ann_2ds: list[str] = field(factory=list, init=False) diff --git a/t4_devkit/schema/v1/sample_annotation.py b/t4_devkit/schema/v1/sample_annotation.py new file mode 100644 index 00000000..021b966a --- /dev/null +++ b/t4_devkit/schema/v1/sample_annotation.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from attrs import converters, define, field, validators + +from t4_devkit.common.converter import to_quaternion +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty +from t4_devkit.typing import Quaternion, Vector3 + +from .annotation_type import AnnotationType +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["SampleAnnotation"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.SAMPLE_ANNOTATION) +class SampleAnnotation(SchemaBase): + """`sample_annotation.json` dataclass. + + A 3D box in the map frame. + + Attributes: + token (str): Unique record identifier. + sample_token (str): Foreign key pointing to the sample. + instance_token (str): Foreign key pointing to the instance. + attribute_tokens (list[str]): Foreign keys pointing to the attributes. + visibility_token (str | None): Foreign key pointing to the visibility level, or None if unknown. + translation (Vector3): Box center [x, y, z] in [m]. + size (Vector3): Box size [width, length, height] in [m]. + rotation (Quaternion): Box orientation as quaternion [w, x, y, z]. + velocity (Vector3 | None): Velocity [vx, vy, vz] in [m/s]. + acceleration (Vector3 | None): Acceleration [ax, ay, az] in [m/s^2]. + num_lidar_pts (int | None): Number of LiDAR points in the box, or None if not counted. + num_radar_pts (int | None): Number of radar points in the box, or None if not counted. + next (str | None): Foreign key pointing to the next annotation of the instance. + prev (str | None): Foreign key pointing to the previous annotation of the instance. + annotation_type (AnnotationType): How the label was made. + provenance (str | None): Vendor, model name and version, or tool. + confidence (float | None): Confidence in [0, 1], or None for manual labels. + + Shortcuts (filled by `T4Tables`): + --------- + category_name (str | None): Name of the category. + """ + + sample_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + instance_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + attribute_tokens: list[str] = field( + validator=validators.deep_iterable( + validators.and_(validators.instance_of(str), impossible_empty()), + validators.instance_of(list), + ) + ) + visibility_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + translation: Vector3 = field(converter=Vector3) + size: Vector3 = field(converter=Vector3) + rotation: Quaternion = field(converter=to_quaternion) + velocity: Vector3 | None = field(converter=converters.optional(Vector3)) + acceleration: Vector3 | None = field(converter=converters.optional(Vector3)) + num_lidar_pts: int | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(int), validators.ge(0)) + ) + ) + num_radar_pts: int | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(int), validators.ge(0)) + ) + ) + next: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + prev: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + annotation_type: AnnotationType = field(converter=AnnotationType) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) + + # shortcuts + category_name: str | None = field(init=False, default=None) diff --git a/t4_devkit/schema/v1/sample_data.py b/t4_devkit/schema/v1/sample_data.py new file mode 100644 index 00000000..3922eda5 --- /dev/null +++ b/t4_devkit/schema/v1/sample_data.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty +from t4_devkit.schema.tables.sensor import SensorModality + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["FileFormat", "SampleData"] + + +@unique +class FileFormat(str, Enum): + """File format of sensor data in T4Dataset 1.0. + + Attributes: + JPG: JPG image, one file per frame (datasets that keep 0.x files). + PNG: PNG image, one file per frame (datasets that keep 0.x files). + PCD: PCD point cloud, one file per frame (datasets that keep 0.x files). + BIN: BIN point cloud, one file per frame (datasets that keep 0.x files). + PCDBIN: PCD.BIN point cloud, one file per frame (datasets that keep 0.x files). + AV1_MP4: AV1 video, one file per camera channel. + PARQUET: Parquet point cloud, one file per LiDAR channel. + """ + + JPG = "jpg" + PNG = "png" + PCD = "pcd" + BIN = "bin" + PCDBIN = "pcd.bin" + AV1_MP4 = "av1_mp4" + PARQUET = "parquet" + + +@define(slots=False) +@SCHEMAS.register(SchemaName.SAMPLE_DATA) +class SampleData(SchemaBase): + """`sample_data.json` dataclass. + + Attributes: + token (str): Unique record identifier. + sample_token (str): Foreign key pointing to the sample (frame) this observation belongs to. + ego_pose_token (str): Foreign key pointing to the ego pose. + sensor_token (str): Foreign key pointing to the sensor. + filename (str): Path to the channel file, or to a per-frame file. + frame_index (int | None): Index of the frame inside the channel file, or None for + per-frame files. + fileformat (FileFormat): Format of the file. + num_points (int | None): Number of points in the frame. LiDAR only. + timestamp (int): Unix time stamp [us]. + is_key_frame (bool): Whether this frame was annotated (per channel). + is_valid (bool): False if the frame has a sensor drop or synchronization failure. + next (str | None): Foreign key pointing to the next sample data of the same channel. + prev (str | None): Foreign key pointing to the previous sample data of the same channel. + + Shortcuts (filled by `T4Tables`): + --------- + modality (SensorModality | None): Modality of the sensor. + channel (str | None): Channel of the sensor. + """ + + sample_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + ego_pose_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + sensor_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + filename: str = field(validator=(validators.instance_of(str), impossible_empty())) + frame_index: int | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(int), validators.ge(0)) + ) + ) + fileformat: FileFormat = field(converter=FileFormat) + num_points: int | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(int), validators.ge(0)) + ) + ) + timestamp: int = field(validator=validators.instance_of(int)) + is_key_frame: bool = field(validator=validators.instance_of(bool)) + is_valid: bool = field(validator=validators.instance_of(bool)) + next: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + prev: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + + # shortcuts + modality: SensorModality | None = field(init=False, default=None) + channel: str | None = field(init=False, default=None) diff --git a/t4_devkit/schema/v1/scenario.py b/t4_devkit/schema/v1/scenario.py new file mode 100644 index 00000000..c0657409 --- /dev/null +++ b/t4_devkit/schema/v1/scenario.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from enum import Enum, unique +from typing import Any + +from attrs import converters, define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .annotation_type import AnnotationType +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["ScenarioStatus", "Taxonomy", "Scenario"] + + +@unique +class ScenarioStatus(str, Enum): + """Whether a scenario is selected or excluded.""" + + WHITELIST = "whitelist" + BLACKLIST = "blacklist" + + +@define +class Taxonomy: + """The vocabulary a scenario is tagged with. + + Attributes: + id (str): Taxonomy ID, e.g. "cod-scene". + version (str): Taxonomy version, e.g. "0.1.1". + """ + + id: str = field(validator=validators.instance_of(str)) + version: str = field(validator=validators.instance_of(str)) + + +@define(slots=False) +@SCHEMAS.register(SchemaName.SCENARIO) +class Scenario(SchemaBase): + """`scenario.json` dataclass. + + A tagged time interval, e.g. from CodSceneClassifier. + + Attributes: + token (str): Unique record identifier. + log_token (str): Foreign key pointing to the log. + start_timestamp (int): Start, Unix time stamp [us] (inclusive). + end_timestamp (int): End, Unix time stamp [us] (exclusive). + key_timestamp (int | None): Key moment [us]. + status (ScenarioStatus | None): Whether the scenario is selected or excluded. + lateral_decision (str | None): Lateral decision, in the taxonomy's vocabulary. + longitudinal_decision (str | None): Longitudinal decision, in the taxonomy's vocabulary. + events (list[str]): Events, in the taxonomy's vocabulary. + scenery (dict[str, Any] | None): Scenery, kept as given by the classifier. + dynamic_entities (dict[str, Any] | None): Dynamic entities, kept as given. + justification (str | None): Justification of the tags. + taxonomy (Taxonomy): The vocabulary the scenario is tagged with. + annotation_type (AnnotationType): How the label was made. + provenance (str | None): Vendor, model name and version, or tool. + confidence (float | None): Confidence in [0, 1], or None for manual labels. + raw (dict[str, Any] | None): Original record. + """ + + log_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + start_timestamp: int = field(validator=validators.instance_of(int)) + end_timestamp: int = field(validator=validators.instance_of(int)) + key_timestamp: int | None = field(validator=validators.optional(validators.instance_of(int))) + status: ScenarioStatus | None = field(converter=converters.optional(ScenarioStatus)) + lateral_decision: str | None = field(validator=validators.optional(validators.instance_of(str))) + longitudinal_decision: str | None = field( + validator=validators.optional(validators.instance_of(str)) + ) + events: list[str] = field( + validator=validators.deep_iterable( + validators.instance_of(str), validators.instance_of(list) + ) + ) + scenery: dict[str, Any] | None = field( + validator=validators.optional(validators.instance_of(dict)) + ) + dynamic_entities: dict[str, Any] | None = field( + validator=validators.optional(validators.instance_of(dict)) + ) + justification: str | None = field(validator=validators.optional(validators.instance_of(str))) + taxonomy: Taxonomy = field( + converter=lambda x: Taxonomy(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(Taxonomy), + ) + annotation_type: AnnotationType = field(converter=AnnotationType) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) + raw: dict[str, Any] | None = field(validator=validators.optional(validators.instance_of(dict))) + + @end_timestamp.validator + def _check_end_timestamp(self, attribute, value: int) -> None: + if value < self.start_timestamp: + raise ValueError(f"end_timestamp {value} is before start_timestamp") diff --git a/t4_devkit/schema/v1/scene.py b/t4_devkit/schema/v1/scene.py new file mode 100644 index 00000000..ba3f5735 --- /dev/null +++ b/t4_devkit/schema/v1/scene.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Scene"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.SCENE) +class Scene(SchemaBase): + """`scene.json` dataclass. + + Attributes: + token (str): Unique record identifier. + name (str): Short string identifier. + description (str | None): Longer description of the scene. + log_token (str): Foreign key pointing to the log the scene was extracted from. + nbr_samples (int): Number of samples in the scene. + first_sample_token (str): Foreign key pointing to the first sample in the scene. + last_sample_token (str): Foreign key pointing to the last sample in the scene. + """ + + name: str = field(validator=validators.instance_of(str)) + description: str | None = field(validator=validators.optional(validators.instance_of(str))) + log_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + nbr_samples: int = field(validator=(validators.instance_of(int), validators.ge(0))) + first_sample_token: str = field(validator=(validators.instance_of(str), impossible_empty())) + last_sample_token: str = field(validator=(validators.instance_of(str), impossible_empty())) diff --git a/t4_devkit/schema/v1/sensor.py b/t4_devkit/schema/v1/sensor.py new file mode 100644 index 00000000..ec0d4be8 --- /dev/null +++ b/t4_devkit/schema/v1/sensor.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +from enum import Enum, unique +from typing import Any + +from attrs import define, field, validators + +from t4_devkit.common.converter import to_quaternion +from t4_devkit.schema.tables.base import SchemaBase +from t4_devkit.schema.tables.sensor import SensorModality +from t4_devkit.typing import CameraDistortion, CameraIntrinsic, Quaternion, Vector3 + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = [ + "DistortionModel", + "VideoFormat", + "Video", + "Camera", + "LidarContainerFormat", + "LidarContainer", + "LidarField", + "Lidar", + "Hardware", + "Sensor", +] + + +@unique +class DistortionModel(str, Enum): + """Camera distortion models, as named in ROS CameraInfo.""" + + PLUMB_BOB = "plumb_bob" + RATIONAL_POLYNOMIAL = "rational_polynomial" + EQUIDISTANT = "equidistant" + NONE = "none" + + @property + def num_coefficients(self) -> int: + """Number of distortion coefficients of this model.""" + return {"plumb_bob": 5, "rational_polynomial": 8, "equidistant": 4, "none": 0}[self.value] + + @classmethod + def from_num_coefficients(cls, num_coefficients: int) -> DistortionModel: + """Return the model with this number of coefficients (each model has a different count).""" + for model in cls: + if model.num_coefficients == num_coefficients: + return model + raise ValueError(f"No distortion model has {num_coefficients} coefficients") + + +@unique +class VideoFormat(str, Enum): + """Container and codec of a camera video.""" + + AV1_MP4 = "av1_mp4" + + +@define +class Video: + """How a camera channel's video is encoded. + + Attributes: + path (str): Path to the video file, relative to the dataset root. + format (VideoFormat): Container and codec. + codec_profile (str): Encoder settings, e.g. "svt-av1 preset6 crf26 intra". + pixel_format (str): Pixel format, e.g. "yuv420p full-range bt601". + bit_depth (int): Bits per channel, 8 or 10. + frame_count (int): Number of frames in the file. + timescale (int): MP4 track timescale, equal to the frame rate so pts == frame_index. + """ + + path: str = field(validator=validators.instance_of(str)) + format: VideoFormat = field(converter=VideoFormat) + codec_profile: str = field(validator=validators.instance_of(str)) + pixel_format: str = field(validator=validators.instance_of(str)) + bit_depth: int = field(validator=validators.in_((8, 10))) + frame_count: int = field(validator=(validators.instance_of(int), validators.ge(0))) + timescale: int = field(validator=(validators.instance_of(int), validators.ge(1))) + + +@define +class Camera: + """Camera sensor. + + Attributes: + width (int): Image width in pixels. + height (int): Image height in pixels. + intrinsic (CameraIntrinsic): 3x3 camera matrix. + distortion (CameraDistortion): Distortion coefficients, following `distortion_model`. + distortion_model (DistortionModel): Distortion model. + is_rectified (bool): Whether the stored images are already rectified. + video (Video | None): How the video is encoded, or None for per-frame image files. + """ + + width: int = field(validator=(validators.instance_of(int), validators.ge(1))) + height: int = field(validator=(validators.instance_of(int), validators.ge(1))) + intrinsic: CameraIntrinsic = field(converter=CameraIntrinsic) + distortion: CameraDistortion = field(converter=CameraDistortion) + distortion_model: DistortionModel = field(converter=DistortionModel) + is_rectified: bool = field(validator=validators.instance_of(bool)) + video: Video | None = field( + converter=lambda x: Video(**x) if isinstance(x, dict) else x, + validator=validators.optional(validators.instance_of(Video)), + ) + + @intrinsic.validator + def _check_intrinsic(self, attribute, value: CameraIntrinsic) -> None: + if value.shape != (3, 3): + raise ValueError( + f"intrinsic must be a 3x3 matrix and cannot be empty for a camera, " + f"got shape {value.shape}" + ) + + @distortion_model.validator + def _check_distortion_model(self, attribute, value: DistortionModel) -> None: + if len(self.distortion) != value.num_coefficients: + raise ValueError( + f"distortion_model {value.value!r} has {value.num_coefficients} coefficients, " + f"got {len(self.distortion)}" + ) + + +@unique +class LidarContainerFormat(str, Enum): + """File format of a LiDAR channel.""" + + PARQUET = "parquet" + PCD = "pcd" + BIN = "bin" + + +@define +class LidarContainer: + """How a LiDAR channel's points are stored. + + Attributes: + format (LidarContainerFormat): File format. + path (str | None): Path to the file, or None for per-frame files. + frame_count (int): Number of frames. + compression (str): Compression profile, e.g. "zstd", or "none". + """ + + format: LidarContainerFormat = field(converter=LidarContainerFormat) + path: str | None = field(validator=validators.optional(validators.instance_of(str))) + frame_count: int = field(validator=(validators.instance_of(int), validators.ge(0))) + compression: str = field(validator=validators.instance_of(str)) + + +@define +class LidarField: + """A point field of a LiDAR channel. + + Attributes: + name (str): Field name, e.g. "x". + dtype (str): Data type, e.g. "float32". + """ + + name: str = field(validator=validators.instance_of(str)) + dtype: str = field(validator=validators.instance_of(str)) + + +@define +class Lidar: + """LiDAR block of a sensor. + + Attributes: + container (LidarContainer): How the points are stored. + fields (list[LidarField]): Point fields in column order. + source_channels (list[str]): LiDAR channels merged into this one; empty if none. + frame_id (str): Coordinate frame of the stored points. + is_undistorted (bool): Whether points are motion and scan compensated. + """ + + container: LidarContainer = field( + converter=lambda x: LidarContainer(**x) if isinstance(x, dict) else x, + validator=validators.instance_of(LidarContainer), + ) + fields: list[LidarField] = field( + converter=lambda x: [LidarField(**v) if isinstance(v, dict) else v for v in x], + validator=validators.deep_iterable( + validators.instance_of(LidarField), validators.instance_of(list) + ), + ) + source_channels: list[str] = field( + validator=validators.deep_iterable( + validators.instance_of(str), validators.instance_of(list) + ) + ) + frame_id: str = field(validator=validators.instance_of(str)) + is_undistorted: bool = field(validator=validators.instance_of(bool)) + + +@define +class Hardware: + """Sensor hardware identity. + + Attributes: + make (str | None): Manufacturer. + model (str | None): Model. + serial_number (str | None): Serial number. + firmware_version (str | None): Firmware version. + """ + + make: str | None = field(validator=validators.optional(validators.instance_of(str))) + model: str | None = field(validator=validators.optional(validators.instance_of(str))) + serial_number: str | None = field(validator=validators.optional(validators.instance_of(str))) + firmware_version: str | None = field(validator=validators.optional(validators.instance_of(str))) + + +@define(slots=False) +@SCHEMAS.register(SchemaName.SENSOR) +class Sensor(SchemaBase): + """`sensor.json` dataclass. + + Attributes: + token (str): Unique record identifier. + channel (str): Sensor channel name. + index (int): Sensor index, unique within the dataset; value of the LiDAR `source_index` + field for points from this sensor. + modality (SensorModality): Sensor modality. + translation (Vector3): Sensor position relative to base_link [m]. + rotation (Quaternion): Sensor orientation relative to base_link, as [w, x, y, z]. + camera (Camera | None): Camera block, set when modality is camera. + lidar (Lidar | None): LiDAR block, set when modality is lidar. + radar (dict[str, Any] | None): Radar block, not defined in 1.0. + hardware (Hardware | None): Sensor hardware identity. + + Shortcuts (filled by `T4Tables`): + --------- + first_sd_token (str | None): Token of the earliest sample data of this sensor. + """ + + channel: str = field(validator=validators.instance_of(str)) + index: int = field( + validator=(validators.instance_of(int), validators.ge(0), validators.le(255)) + ) + modality: SensorModality = field(converter=SensorModality) + translation: Vector3 = field(converter=Vector3) + rotation: Quaternion = field(converter=to_quaternion) + camera: Camera | None = field( + converter=lambda x: Camera(**x) if isinstance(x, dict) else x, + validator=validators.optional(validators.instance_of(Camera)), + ) + lidar: Lidar | None = field( + converter=lambda x: Lidar(**x) if isinstance(x, dict) else x, + validator=validators.optional(validators.instance_of(Lidar)), + ) + radar: dict[str, Any] | None = field( + validator=validators.optional(validators.instance_of(dict)) + ) + hardware: Hardware | None = field( + converter=lambda x: Hardware(**x) if isinstance(x, dict) else x, + validator=validators.optional(validators.instance_of(Hardware)), + ) + + # shortcuts + first_sd_token: str | None = field(init=False, default=None) + + def __attrs_post_init__(self) -> None: + """Post-initialization validation: only the block matching `modality` is set.""" + blocks = { + SensorModality.CAMERA: self.camera, + SensorModality.LIDAR: self.lidar, + SensorModality.RADAR: self.radar, + } + for modality, block in blocks.items(): + if (block is not None) != (modality == self.modality): + raise ValueError( + f"Sensor {self.channel!r} has modality {self.modality.value!r}: " + f"only the {self.modality.value!r} block must be set." + ) diff --git a/t4_devkit/schema/v1/traffic_light.py b/t4_devkit/schema/v1/traffic_light.py new file mode 100644 index 00000000..96ef746c --- /dev/null +++ b/t4_devkit/schema/v1/traffic_light.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase, impossible_empty + +from .annotation_type import AnnotationType +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = [ + "TrafficLightColor", + "TrafficLightShape", + "TrafficLightStatus", + "TrafficLightElement", + "TrafficLight", +] + + +@unique +class TrafficLightColor(str, Enum): + """Bulb color, as in Autoware TrafficLightElement.""" + + RED = "red" + AMBER = "amber" + GREEN = "green" + WHITE = "white" + UNKNOWN = "unknown" + + +@unique +class TrafficLightShape(str, Enum): + """Bulb shape, as in Autoware TrafficLightElement.""" + + CIRCLE = "circle" + LEFT_ARROW = "left_arrow" + RIGHT_ARROW = "right_arrow" + UP_ARROW = "up_arrow" + UP_LEFT_ARROW = "up_left_arrow" + UP_RIGHT_ARROW = "up_right_arrow" + DOWN_ARROW = "down_arrow" + DOWN_LEFT_ARROW = "down_left_arrow" + DOWN_RIGHT_ARROW = "down_right_arrow" + CROSS = "cross" + UNKNOWN = "unknown" + + +@unique +class TrafficLightStatus(str, Enum): + """Bulb status, as in Autoware TrafficLightElement.""" + + SOLID_ON = "solid_on" + SOLID_OFF = "solid_off" + FLASHING = "flashing" + UNKNOWN = "unknown" + + +@define +class TrafficLightElement: + """The state of one bulb, as in Autoware TrafficLightElement. + + Attributes: + color (TrafficLightColor): Bulb color. + shape (TrafficLightShape): Bulb shape. + status (TrafficLightStatus): Bulb status. + confidence (float | None): Confidence in [0, 1]. + """ + + color: TrafficLightColor = field(converter=TrafficLightColor) + shape: TrafficLightShape = field(converter=TrafficLightShape) + status: TrafficLightStatus = field(converter=TrafficLightStatus) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) + + +@define(slots=False) +@SCHEMAS.register(SchemaName.TRAFFIC_LIGHT) +class TrafficLight(SchemaBase): + """`traffic_light.json` dataclass. + + The state of one traffic light at one time, keyed by its lanelet2 regulatory element + (= Autoware `TrafficLightGroup.traffic_light_group_id`). + + Attributes: + token (str): Unique record identifier. + timestamp (int): Unix time stamp [us]. + regulatory_element_id (int): lanelet2 traffic light regulatory element ID. + elements (list[TrafficLightElement]): Bulb states. + object_ann_token (str | None): Foreign key pointing to the object annotation the state + was read from. + instance_token (str | None): Foreign key pointing to the lamp instance, for states read + from 2D labels. + annotation_type (AnnotationType): How the label was made. + provenance (str | None): Vendor, recognition model and version, or "v2x". + confidence (float | None): Confidence in [0, 1], or None for manual labels. + """ + + timestamp: int = field(validator=validators.instance_of(int)) + regulatory_element_id: int = field(validator=validators.instance_of(int)) + elements: list[TrafficLightElement] = field( + converter=lambda x: [TrafficLightElement(**v) if isinstance(v, dict) else v for v in x], + validator=validators.deep_iterable( + validators.instance_of(TrafficLightElement), validators.instance_of(list) + ), + ) + object_ann_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + instance_token: str | None = field( + validator=validators.optional( + validators.and_(validators.instance_of(str), impossible_empty()) + ) + ) + annotation_type: AnnotationType = field(converter=AnnotationType) + provenance: str | None = field(validator=validators.optional(validators.instance_of(str))) + confidence: float | None = field( + validator=validators.optional( + validators.and_( + validators.instance_of((int, float)), validators.ge(0), validators.le(1) + ) + ) + ) diff --git a/t4_devkit/schema/v1/vehicle.py b/t4_devkit/schema/v1/vehicle.py new file mode 100644 index 00000000..a52297ca --- /dev/null +++ b/t4_devkit/schema/v1/vehicle.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["Vehicle"] + + +@define(slots=False) +@SCHEMAS.register(SchemaName.VEHICLE) +class Vehicle(SchemaBase): + """`vehicle.json` dataclass. + + Attributes: + token (str): Unique record identifier. + name (str | None): Vehicle name, e.g. "j6_gen2_04". + model (str): Vehicle model. + length (float): Length [m]. + width (float): Width [m]. + height (float): Height [m]. + wheel_base (float): Wheel base [m]. + rear_axle_to_center (float | None): Distance from the rear axle to the vehicle center [m]. + sensor_rig_id (str | None): Sensor rig ID. + """ + + name: str | None = field(validator=validators.optional(validators.instance_of(str))) + model: str = field(validator=validators.instance_of(str)) + length: float = field(validator=(validators.instance_of((int, float)), validators.gt(0))) + width: float = field(validator=(validators.instance_of((int, float)), validators.gt(0))) + height: float = field(validator=(validators.instance_of((int, float)), validators.gt(0))) + wheel_base: float = field(validator=(validators.instance_of((int, float)), validators.gt(0))) + rear_axle_to_center: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + sensor_rig_id: str | None = field(validator=validators.optional(validators.instance_of(str))) diff --git a/t4_devkit/schema/v1/vehicle_state.py b/t4_devkit/schema/v1/vehicle_state.py new file mode 100644 index 00000000..03694bdf --- /dev/null +++ b/t4_devkit/schema/v1/vehicle_state.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import converters, define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase +from t4_devkit.schema.tables.vehicle_state import Indicators + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["ShiftState", "VehicleStateSource", "VehicleState"] + + +@unique +class ShiftState(str, Enum): + """Gear shift state, as in Autoware GearReport.""" + + PARK = "PARK" + REVERSE = "REVERSE" + NEUTRAL = "NEUTRAL" + DRIVE = "DRIVE" + LOW = "LOW" + NONE = "NONE" + UNKNOWN = "unknown" + + +@unique +class VehicleStateSource(str, Enum): + """Where vehicle signals came from.""" + + CAN = "can" + VEHICLE_INTERFACE = "vehicle_interface" + IMU = "imu" + + +@define(slots=False) +@SCHEMAS.register(SchemaName.VEHICLE_STATE) +class VehicleState(SchemaBase): + """`vehicle_state.json` dataclass. + + Measured vehicle signals at their native rate (not resampled). + + Attributes: + token (str): Unique record identifier. + timestamp (int): Unix time stamp [us]. + accel_pedal (float | None): Accelerator pedal position. + brake_pedal (float | None): Brake pedal position. + steer_pedal (float | None): Steering position. + steering_tire_angle (float | None): Steering tire angle [rad]. + steering_wheel_angle (float | None): Steering wheel angle [rad]. + shift_state (ShiftState | None): Gear shift state. + indicators (Indicators | None): Turn indicator and hazard state. + longitudinal_velocity (float | None): Longitudinal velocity [m/s]. + lateral_velocity (float | None): Lateral velocity [m/s]. + heading_rate (float | None): Heading rate [rad/s]. + source (VehicleStateSource): Where the signals came from. + """ + + timestamp: int = field(validator=validators.instance_of(int)) + accel_pedal: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + brake_pedal: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + steer_pedal: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + steering_tire_angle: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + steering_wheel_angle: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + shift_state: ShiftState | None = field(converter=converters.optional(ShiftState)) + indicators: Indicators | None = field( + converter=lambda x: Indicators(**x) if isinstance(x, dict) else x, + validator=validators.optional(validators.instance_of(Indicators)), + ) + longitudinal_velocity: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + lateral_velocity: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + heading_rate: float | None = field( + validator=validators.optional(validators.instance_of((int, float))) + ) + source: VehicleStateSource = field(converter=VehicleStateSource) diff --git a/t4_devkit/schema/v1/visibility.py b/t4_devkit/schema/v1/visibility.py new file mode 100644 index 00000000..13b52263 --- /dev/null +++ b/t4_devkit/schema/v1/visibility.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from enum import Enum, unique + +from attrs import define, field, validators + +from t4_devkit.schema.tables.base import SchemaBase + +from .name import SchemaName +from .registry import SCHEMAS + +__all__ = ["VisibilityLevel", "Visibility"] + + +@unique +class VisibilityLevel(str, Enum): + """Visibility levels of an annotation. + + Attributes: + FULL: The annotation is fully visible. + MOST: The annotation is mostly visible, more than 50%. + PARTIAL: The annotation is partially visible, between 10% and 50%. + NONE: The annotation is not visible. + """ + + FULL = "full" + MOST = "most" + PARTIAL = "partial" + NONE = "none" + + +@define(slots=False) +@SCHEMAS.register(SchemaName.VISIBILITY) +class Visibility(SchemaBase): + """A dataclass to represent schema table of `visibility.json` in T4Dataset 1.0. + + Attributes: + token (str): Unique record identifier. + level (VisibilityLevel): Visibility level. + description (str | None): Description of the level. + """ + + level: VisibilityLevel = field(converter=VisibilityLevel) + description: str | None = field(validator=validators.optional(validators.instance_of(str))) diff --git a/t4_devkit/schema/v1/writer.py b/t4_devkit/schema/v1/writer.py new file mode 100644 index 00000000..dc8a605e --- /dev/null +++ b/t4_devkit/schema/v1/writer.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import importlib.metadata +import json +import shutil +from pathlib import Path +from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence + +from t4_devkit.common.io import file_sha256, save_pointcloud, save_video +from t4_devkit.common.serialize import serialize_dataclass, serialize_dataclasses + +from .manifest import MANIFEST_FILENAME, Capabilities, Manifest +from .name import SchemaName + +if TYPE_CHECKING: + import numpy as np + + from t4_devkit.typing import PathLike + +__all__ = ["DatasetWriter"] + + +class DatasetWriter: + """Write a T4Dataset 1.0 dataset folder: sensor data, then tables and `manifest.json`. + + The writer writes files; the caller decides their content. Data files go first + (`write_lidar`, `write_camera`, `copy`), then `save` writes the tables and the manifest, with + the hash of every file in the folder. + + Examples: + >>> writer = DatasetWriter("out//") + >>> path, frame_count = writer.write_lidar("LIDAR_TOP", frames, fields) + >>> writer.copy("in/map/lanelet2_map.osm", "map/lanelet2_map.osm") + >>> manifest = writer.save(tables, manifest_info) + """ + + def __init__(self, output_root: PathLike) -> None: + """ + Args: + output_root (PathLike): Dataset root to create, i.e. `/`. + + Raises: + FileExistsError: If the folder already exists: a dataset is never written in place. + """ + self.output_root = Path(output_root) + if self.output_root.exists(): + raise FileExistsError(f"{self.output_root} already exists") + self.output_root.mkdir(parents=True) + + def write_lidar( + self, + channel: str, + frames: Iterable[Mapping[str, np.ndarray]], + fields: Sequence[tuple[str, str]], + **kwargs: Any, + ) -> tuple[str, int]: + """Write the frames of a LiDAR channel to `data//.parquet`. + + Args: + channel (str): Channel name. + frames (Iterable[Mapping[str, np.ndarray]]): Point fields per frame. + fields (Sequence[tuple[str, str]]): (name, dtype) of each column. + **kwargs: Options of `save_pointcloud`. + + Returns: + The path relative to the dataset root and the number of frames. + + Raises: + FileExistsError: If the channel's file is already written. + """ + path = self._data_path(channel, "parquet") + frame_count = save_pointcloud(self.output_root / path, frames, fields, **kwargs) + return path, frame_count + + def write_camera( + self, + channel: str, + frames: Iterable[np.ndarray], + frame_rate: int, + **kwargs: Any, + ) -> dict[str, Any]: + """Write the frames of a camera channel to `data//.mp4`. + + Args: + channel (str): Channel name. + frames (Iterable[np.ndarray]): RGB images. + frame_rate (int): Frames per second. + **kwargs: Options of `save_video`. + + Returns: + The values of `sensor.camera.video`, `path` included. + + Raises: + FileExistsError: If the channel's file is already written. + """ + path = self._data_path(channel, "mp4") + video = save_video(self.output_root / path, frames, frame_rate, **kwargs) + return {"path": path} | video + + def copy(self, source: PathLike, path: str) -> None: + """Copy a file or folder as it is, e.g. the map, `input_bag/` or per-frame images. + + Args: + source (PathLike): File or folder to copy. + path (str): Destination relative to the dataset root. + """ + destination = self.output_root / path + destination.parent.mkdir(parents=True, exist_ok=True) + if Path(source).is_dir(): + shutil.copytree(source, destination) + else: + shutil.copy2(source, destination) + + def save( + self, + tables: Mapping[SchemaName, Sequence[Any]], + manifest_info: Mapping[str, Any], + ) -> Manifest: + """Write the tables and `manifest.json`. + + Every required table is written, and every optional table with records. The manifest + lists them and every file under `data/`, `map/` and `input_bag/` with its hash and size. + Its `capabilities` are computed from the tables unless given, and t4-devkit is added to + its `generators`. + + Args: + tables (Mapping[SchemaName, Sequence[Any]]): 1.0 records per table. + manifest_info (Mapping[str, Any]): The other `manifest.json` fields: identity, + provenance, migration notes, ... + + Returns: + The manifest written. + """ + table_entries = {} + for table in SchemaName: + records = tables.get(table, []) + if not records and table.is_optional(): + continue + path = f"annotation/{table.filename}" + self._write_json(path, serialize_dataclasses(list(records))) + table_entries[table.value] = {"path": path, "sha256": self._sha256(path)} + + assets = [ + { + "path": path, + "sha256": self._sha256(path), + "bytes": (self.output_root / path).stat().st_size, + } + for path in self._asset_paths() + ] + generator = {"name": "t4-devkit", "version": importlib.metadata.version("t4-devkit")} + generators = list(manifest_info.get("generators", [])) + if generator not in generators: + generators.append(generator) + capabilities = manifest_info.get("capabilities") or Capabilities.from_tables(tables) + + data = dict(manifest_info) + data.update( + generators=generators, + capabilities=capabilities, + tables=table_entries, + assets=assets, + ) + manifest = Manifest(**data) + self._write_json(MANIFEST_FILENAME, serialize_dataclass(manifest)) + return manifest + + def _data_path(self, channel: str, extension: str) -> str: + path = f"data/{channel}/{channel}.{extension}" + if (self.output_root / path).exists(): + raise FileExistsError(f"{path} is already written: a channel has one file") + (self.output_root / path).parent.mkdir(parents=True, exist_ok=True) + return path + + def _asset_paths(self) -> list[str]: + """Return every file under the asset folders, relative to the dataset root, sorted.""" + return sorted( + filepath.relative_to(self.output_root).as_posix() + for directory in Manifest.ASSET_DIRS + for filepath in (self.output_root / directory).rglob("*") + if filepath.is_file() + ) + + def _write_json(self, path: str, data: Any) -> None: + filepath = self.output_root / path + filepath.parent.mkdir(parents=True, exist_ok=True) + filepath.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n") + + def _sha256(self, path: str) -> str: + return file_sha256(self.output_root / path) diff --git a/t4_devkit/tier4.py b/t4_devkit/tier4.py index e9cfa290..fe179288 100644 --- a/t4_devkit/tier4.py +++ b/t4_devkit/tier4.py @@ -15,37 +15,38 @@ from t4_devkit.common.geometry import is_box_in_image from t4_devkit.dataclass import Box2D, Box3D, SemanticLabel, Shape, ShapeType from t4_devkit.helper import RenderingHelper, TimeseriesHelper -from t4_devkit.schema import SchemaName, SensorModality, VisibilityLevel, build_schema -from t4_devkit.schema.compatibility import fix_category_table +from t4_devkit.schema import SensorModality, VisibilityLevel +from t4_devkit.schema.v1 import SchemaName, load_dataset if TYPE_CHECKING: + from t4_devkit.schema import SchemaTable from t4_devkit.typing import CameraIntrinsicLike, PathLike, Vector3 from .dataclass import BoxLike - from .schema import ( + from .schema.v1 import ( Attribute, - CalibratedSensor, + Caption, Category, EgoPose, Instance, Keypoint, - LidarSeg, Log, Map, ObjectAnn, + Route, Sample, SampleAnnotation, SampleData, + Scenario, Scene, - SchemaTable, Sensor, - SurfaceAnn, TrafficLight, + Vehicle, VehicleState, Visibility, ) -__all__ = ["DBMetadata", "load_metadata", "load_table", "Tier4", "T4Devkit"] +__all__ = ["DBMetadata", "load_metadata", "Tier4", "T4Devkit"] @define @@ -96,29 +97,6 @@ def load_metadata(db_root: PathLike, revision: str | None = None) -> DBMetadata: return DBMetadata(data_root=data_root, dataset_id=dataset_id, version=version) -def load_table(annotation_dir: str, schema: SchemaName) -> list[SchemaTable]: - """Load schema table from a JSON file. - - If the schema is optional and there is no corresponding JSON file in dataset, - returns empty list. - - Args: - annotation_dir (str): Path to the directory of JSON annotation schema files. - schema (SchemaName): An enum member of `SchemaName`. - - Returns: - Loaded table data saved in `.json`. - """ - filepath = osp.join(annotation_dir, schema.filename) - if not osp.exists(filepath) and schema.is_optional(): - return [] - - if not osp.exists(filepath): - raise FileNotFoundError(f"{schema.value} is mandatory.") - - return build_schema(schema, filepath) - - class T4Devkit: """Database class for T4 dataset to help query and retrieve information from the database.""" @@ -140,26 +118,16 @@ def __init__( >>> from t4_devkit import T4Devkit >>> t4 = T4Devkit("data/tier4") ====== - Loading T4 tables in `annotation`... - Reverse indexing... - Done reverse indexing in 0.010 seconds. - ====== - 21 category - 8 attribute - 4 visibility - 31 instance - 7 sensor - 7 calibrated_sensor - 2529 ego_pose - 1 log - 1 scene + Loading T4 tables... 88 sample 2529 sample_data - 1919 sample_annotation - 0 object_ann - 0 surface_ann - 0 keypoint + 2529 ego_pose + 7 sensor + 1 scene + 1 log + 1 vehicle 1 map + ... Done loading in 0.046 seconds. ====== @@ -169,48 +137,41 @@ def __init__( if not osp.exists(self.data_root): raise FileNotFoundError(f"Database directory is not found: {self.data_root}") - if self.version is None: + if self._metadata.version is None: warnings.warn( - f"DatasetID: {self.dataset_id} does't contain any versions.", DeprecationWarning + f"DatasetID: {self._metadata.dataset_id} does't contain any versions.", + DeprecationWarning, ) start_time = time.time() if verbose: print("======\nLoading T4 tables...") - # assign tables explicitly - self.attribute: list[Attribute] = load_table(self.annotation_dir, SchemaName.ATTRIBUTE) - self.calibrated_sensor: list[CalibratedSensor] = load_table( - self.annotation_dir, SchemaName.CALIBRATED_SENSOR - ) - self.category: list[Category] = load_table(self.annotation_dir, SchemaName.CATEGORY) - self.category = fix_category_table(self.category) - - self.ego_pose: list[EgoPose] = load_table(self.annotation_dir, SchemaName.EGO_POSE) - self.instance: list[Instance] = load_table(self.annotation_dir, SchemaName.INSTANCE) - self.keypoint: list[Keypoint] = load_table(self.annotation_dir, SchemaName.KEYPOINT) - self.lidarseg: list[LidarSeg] = load_table(self.annotation_dir, SchemaName.LIDARSEG) - self.log: list[Log] = load_table(self.annotation_dir, SchemaName.LOG) - self.map: list[Map] = load_table(self.annotation_dir, SchemaName.MAP) - self.object_ann: list[ObjectAnn] = load_table(self.annotation_dir, SchemaName.OBJECT_ANN) - self.sample_annotation: list[SampleAnnotation] = load_table( - self.annotation_dir, SchemaName.SAMPLE_ANNOTATION - ) - self.sample_data: list[SampleData] = load_table(self.annotation_dir, SchemaName.SAMPLE_DATA) - self.sample: list[Sample] = load_table(self.annotation_dir, SchemaName.SAMPLE) - self.scene: list[Scene] = load_table(self.annotation_dir, SchemaName.SCENE) - self.sensor: list[Sensor] = load_table(self.annotation_dir, SchemaName.SENSOR) - self.surface_ann: list[SurfaceAnn] = load_table(self.annotation_dir, SchemaName.SURFACE_ANN) - self.vehicle_state: list[VehicleState] = load_table( - self.annotation_dir, SchemaName.VEHICLE_STATE - ) - self.visibility: list[Visibility] = load_table(self.annotation_dir, SchemaName.VISIBILITY) - self.traffic_light: list[TrafficLight] = load_table( - self.annotation_dir, SchemaName.TRAFFIC_LIGHT - ) + # 0.x datasets are converted to 1.0 records on loading. + self._tables = load_dataset(self.data_root) + tables = self._tables.tables - # make reverse indexes for common lookups - self.__make_reverse_index__(verbose) + # assign tables explicitly + self.sample: list[Sample] = tables[SchemaName.SAMPLE] + self.sample_data: list[SampleData] = tables[SchemaName.SAMPLE_DATA] + self.ego_pose: list[EgoPose] = tables[SchemaName.EGO_POSE] + self.sensor: list[Sensor] = tables[SchemaName.SENSOR] + self.scene: list[Scene] = tables[SchemaName.SCENE] + self.log: list[Log] = tables[SchemaName.LOG] + self.vehicle: list[Vehicle] = tables[SchemaName.VEHICLE] + self.map: list[Map] = tables[SchemaName.MAP] + self.vehicle_state: list[VehicleState] = tables[SchemaName.VEHICLE_STATE] + self.category: list[Category] = tables[SchemaName.CATEGORY] + self.attribute: list[Attribute] = tables[SchemaName.ATTRIBUTE] + self.visibility: list[Visibility] = tables[SchemaName.VISIBILITY] + self.instance: list[Instance] = tables[SchemaName.INSTANCE] + self.sample_annotation: list[SampleAnnotation] = tables[SchemaName.SAMPLE_ANNOTATION] + self.object_ann: list[ObjectAnn] = tables[SchemaName.OBJECT_ANN] + self.keypoint: list[Keypoint] = tables[SchemaName.KEYPOINT] + self.traffic_light: list[TrafficLight] = tables[SchemaName.TRAFFIC_LIGHT] + self.route: list[Route] = tables[SchemaName.ROUTE] + self.scenario: list[Scenario] = tables[SchemaName.SCENARIO] + self.caption: list[Caption] = tables[SchemaName.CAPTION] if verbose: for schema in SchemaName: @@ -229,13 +190,16 @@ def data_root(self) -> str: @property def dataset_id(self) -> str: - """Return the dataset ID.""" - return self._metadata.dataset_id + """Return the dataset ID: from `manifest.json`, or the folder name for 0.x.""" + manifest = self._tables.manifest + return manifest.dataset_id if manifest is not None else self._metadata.dataset_id @property def version(self) -> str | None: - """Return the dataset version, or None if it is failed to lookup.""" - return self._metadata.version + """Return the dataset version: from `manifest.json`, or the folder name for 0.x (None if + there is no version folder).""" + manifest = self._tables.manifest + return str(manifest.version) if manifest is not None else self._metadata.version @property def annotation_dir(self) -> str: @@ -252,91 +216,6 @@ def bag_dir(self) -> str: """Return the path to ROS bag directory.""" return osp.join(self.data_root, "input_bag") - def __make_reverse_index__(self, verbose: bool) -> None: - """De-normalize database to create reverse indices for common cases. - - Args: - verbose (bool): Whether to display outputs. - - Raises: - ValueError: Expecting `map` table has `log_tokens` key. - """ - start_time = time.time() - if verbose: - print("Reverse indexing...") - - self._token2idx: dict[str, dict[str, int]] = { - schema.value: { - table.token: idx for idx, table in enumerate(self.get_table(schema.value)) - } - for schema in SchemaName - } - - self._label2id: dict[str, int] = { - category.name: category.index for category in self.category - } - - # add shortcuts - for record in self.sample_annotation: - instance: Instance = self.get("instance", record.instance_token) - category: Category = self.get("category", instance.category_token) - record.category_name = category.name - - for record in self.object_ann: - category: Category = self.get("category", record.category_token) - record.category_name = category.name - - for record in self.surface_ann: - if record.category_token == "": # NOTE: Some database contains this case - warnings.warn(f"Category token is empty for surface ann: {record.token}") - continue - category: Category = self.get("category", record.category_token) - record.category_name = category.name - - registered_channels: list[str] = [] - for record in self.sample_data: - cs_record: CalibratedSensor = self.get( - "calibrated_sensor", record.calibrated_sensor_token - ) - sensor_record: Sensor = self.get("sensor", cs_record.sensor_token) - record.modality = sensor_record.modality - record.channel = sensor_record.channel - # set first sample data token to the corresponding sensor channel, - # as premise for sample data is ordered by time stamp order. - if sensor_record.channel not in registered_channels: - sensor_record.first_sd_token = record.token - registered_channels.append(sensor_record.channel) - - # set sample data - if record.is_key_frame: - sample_record: Sample = self.get("sample", record.sample_token) - sample_record.data[record.channel] = record.token - - for ann_record in self.sample_annotation: - sample_record: Sample = self.get("sample", ann_record.sample_token) - sample_record.ann_3ds.append(ann_record.token) - - for ann_record in self.object_ann: - sd_record: SampleData = self.get("sample_data", ann_record.sample_data_token) - sample_record: Sample = self.get("sample", sd_record.sample_token) - sample_record.ann_2ds.append(ann_record.token) - - for ann_record in self.surface_ann: - sd_record: SampleData = self.get("sample_data", ann_record.sample_data_token) - sample_record: Sample = self.get("sample", sd_record.sample_token) - sample_record.surface_anns.append(ann_record.token) - - log_to_map: dict[str, str] = {} - for map_record in self.map: - for log_token in map_record.log_tokens: - log_to_map[log_token] = map_record.token - for log_record in self.log: - log_record.map_token = log_to_map[log_record.token] - - if verbose: - elapsed_time = time.time() - start_time - print(f"Done reverse indexing in {elapsed_time:.3f} seconds.\n======") - def get_table(self, schema: str | SchemaName) -> list[SchemaTable]: """Return the list of dataclasses corresponding to the schema table. @@ -346,7 +225,7 @@ def get_table(self, schema: str | SchemaName) -> list[SchemaTable]: Returns: List of dataclasses. """ - return getattr(self, SchemaName(schema)) + return self._tables.tables[SchemaName(schema)] def get(self, schema: str | SchemaName, token: str) -> SchemaTable: """Return a record identified by the associated token. @@ -358,7 +237,7 @@ def get(self, schema: str | SchemaName, token: str) -> SchemaTable: Returns: Table record of the corresponding token. """ - return self.get_table(schema)[self.get_idx(schema, token)] + return self._tables.get(schema, token) def get_idx(self, schema: str | SchemaName, token: str) -> int: """Return the index of the record in a table in constant runtime. @@ -370,12 +249,7 @@ def get_idx(self, schema: str | SchemaName, token: str) -> int: Returns: The index of the record in table. """ - schema = SchemaName(schema) - if self._token2idx.get(schema) is None: - raise KeyError(f"{schema} is not registered.") - if self._token2idx[schema].get(token) is None: - raise KeyError(f"{token} is not registered in {schema}.") - return self._token2idx[schema][token] + return self._tables.get_idx(schema, token) def get_sample_data_path(self, sample_data_token: str) -> str: """Return the file path to a raw data recorded in `sample_data`. @@ -416,17 +290,14 @@ def get_sample_data( """ # Retrieve sensor & pose records sd_record: SampleData = self.get("sample_data", sample_data_token) - cs_record: CalibratedSensor = self.get( - "calibrated_sensor", sd_record.calibrated_sensor_token - ) - sensor_record: Sensor = self.get("sensor", cs_record.sensor_token) + sensor_record: Sensor = self.get("sensor", sd_record.sensor_token) pose_record: EgoPose = self.get("ego_pose", sd_record.ego_pose_token) data_path = self.get_sample_data_path(sample_data_token) if sensor_record.modality == SensorModality.CAMERA: - cam_intrinsic = cs_record.camera_intrinsic - img_size = (sd_record.width, sd_record.height) + cam_intrinsic = sensor_record.camera.intrinsic + img_size = (sensor_record.camera.width, sensor_record.camera.height) else: cam_intrinsic = None img_size = None @@ -462,8 +333,8 @@ def get_sample_data( if as_sensor_coord: # Move box to sensor coord system. - box.translate(-cs_record.translation) - box.rotate(cs_record.rotation.inverse) + box.translate(-sensor_record.translation) + box.rotate(sensor_record.rotation.inverse) box.frame_id = sensor_record.channel if sensor_record.modality == SensorModality.CAMERA and not is_box_in_image( @@ -513,7 +384,6 @@ def get_box3d(self, sample_annotation_token: str, *, future_seconds: float = 0.0 ann: SampleAnnotation = self.get("sample_annotation", sample_annotation_token) instance: Instance = self.get("instance", ann.instance_token) sample: Sample = self.get("sample", ann.sample_token) - visibility: Visibility = self.get("visibility", ann.visibility_token) # semantic label semantic_label = self.get_semantic_label( @@ -534,10 +404,10 @@ def get_box3d(self, sample_annotation_token: str, *, future_seconds: float = 0.0 rotation=ann.rotation, shape=shape, velocity=velocity, - confidence=1.0, - uuid=instance.token, # TODO(ktro2828): extract uuid from `instance_name`. + confidence=self._box_confidence(ann), + uuid=instance.token, # TODO(ktro2828): extract uuid from `instance.name`. num_points=ann.num_lidar_pts, - visibility=visibility.level, + visibility=self._box_visibility(ann), ) if future_seconds > 0.0: @@ -562,7 +432,6 @@ def get_box2d(self, object_ann_token: str) -> Box2D: Instantiated Box2D. """ ann: ObjectAnn = self.get("object_ann", object_ann_token) - instance: Instance = self.get("instance", ann.instance_token) sample_data: SampleData = self.get("sample_data", ann.sample_data_token) semantic_label = self.get_semantic_label( @@ -575,13 +444,16 @@ def get_box2d(self, object_ann_token: str) -> Box2D: frame_id=sample_data.channel, semantic_label=semantic_label, roi=ann.bbox, - confidence=1.0, - uuid=instance.token, # TODO(ktro2828): extract uuid from `instance_name`. + confidence=self._box_confidence(ann), + # regions without an instance (road, sky) have no uuid + uuid=ann.instance_token, # TODO(ktro2828): extract uuid from `instance.name`. ) def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> list[Box3D]: """Rerun a list of Box3D classes for all annotations of a particular `sample_data` record. - It the `sample_data` is a keyframe, this returns annotations for the corresponding `sample`. + It the `sample` of the `sample_data` has a key frame, this returns its annotations. + Otherwise, boxes are interpolated between the next key sample and the previous one; after + the last key sample, the boxes of the last one are returned. Args: sample_data_token (str): Token of `sample_data`. @@ -592,18 +464,27 @@ def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> """ # Retrieve sensor & pose records sd_record: SampleData = self.get("sample_data", sample_data_token) - curr_sample_record: Sample = self.get("sample", sd_record.sample_token) + sample_record: Sample = self.get("sample", sd_record.sample_token) + curr_sample_record = self._key_sample(sample_record.token, forward=True) + after_last_key_sample = curr_sample_record is None + if after_last_key_sample: + curr_sample_record = self._key_sample(sample_record.token, forward=False) + if curr_sample_record is None: + return [] + prev_sample_record = ( + None + if sample_record.has_key_frame or after_last_key_sample + else self._key_sample(curr_sample_record.prev, forward=False) + ) - if curr_sample_record.prev == "" or sd_record.is_key_frame: - # If no previous annotations available, or if sample_data is keyframe just return the current ones. + if prev_sample_record is None: + # A key sample, or no key sample on one side: just return the key sample's boxes. boxes = [ self.get_box3d(token, future_seconds=future_seconds) for token in curr_sample_record.ann_3ds ] else: - prev_sample_record: Sample = self.get("sample", curr_sample_record.prev) - curr_ann_recs: list[SampleAnnotation] = [ self.get("sample_annotation", token) for token in curr_sample_record.ann_3ds ] @@ -649,7 +530,6 @@ def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> instance.category_token, curr_ann.attribute_tokens ) velocity = self.box_velocity(curr_ann.token) - visibility: Visibility = self.get("visibility", curr_ann.visibility_token) box = Box3D( unix_time=t, @@ -659,10 +539,10 @@ def get_box3ds(self, sample_data_token: str, *, future_seconds: float = 0.0) -> rotation=rotation, shape=Shape(ShapeType.BOUNDING_BOX, curr_ann.size), velocity=velocity, - confidence=1.0, - uuid=instance.token, # TODO(ktro2828): extract uuid from `instance_name`. + confidence=self._box_confidence(curr_ann), + uuid=instance.token, # TODO(ktro2828): extract uuid from `instance.name`. num_points=curr_ann.num_lidar_pts, - visibility=visibility.level, + visibility=self._box_visibility(curr_ann), ) else: # If not, simply grab the current annotation. @@ -710,8 +590,8 @@ def box_velocity(self, sample_annotation_token: str, max_time_diff: float = 1.5) if current.velocity is not None: return current.velocity - has_prev = current.prev != "" - has_next = current.next != "" + has_prev = current.prev is not None + has_next = current.next is not None # Cannot estimate velocity for a single annotation. if not has_prev and not has_next: @@ -814,6 +694,31 @@ def render_pointcloud( show_map=show_map, ) + def _key_sample(self, sample_token: str | None, *, forward: bool) -> Sample | None: + """Return the first sample with a key frame from a sample (included), following `next` or + `prev`, or None. + """ + while sample_token is not None: + sample: Sample = self.get("sample", sample_token) + if sample.has_key_frame: + return sample + sample_token = sample.next if forward else sample.prev + return None + + def _box_visibility(self, ann: SampleAnnotation) -> VisibilityLevel: + """Return the visibility of a 3D box; `UNAVAILABLE` when the annotation's visibility is + unknown. + """ + if ann.visibility_token is None: + return VisibilityLevel.UNAVAILABLE + visibility: Visibility = self.get("visibility", ann.visibility_token) + return VisibilityLevel(visibility.level.value) + + @staticmethod + def _box_confidence(ann: SampleAnnotation | ObjectAnn) -> float: + """Return the confidence of a box: the label's, or 1.0 for labels without one (manual).""" + return 1.0 if ann.confidence is None else ann.confidence + @deprecated("Tier4 is deprecated. Use T4Devkit instead.") class Tier4(T4Devkit): diff --git a/t4_devkit/viewer/viewer.py b/t4_devkit/viewer/viewer.py index 72a73b5e..3fdd41ef 100644 --- a/t4_devkit/viewer/viewer.py +++ b/t4_devkit/viewer/viewer.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from t4_devkit.dataclass import Box2D, Box3D, Future, PointCloudLike - from t4_devkit.schema import CalibratedSensor, EgoPose, Sensor + from t4_devkit.schema.v1 import EgoPose, Sensor from t4_devkit.typing import ( CameraIntrinsicLike, NDArrayU8, @@ -545,18 +545,12 @@ def _render_ego_without_schema( rr.log(entity_path, rr.GeoPoints(lat_lon=(latitude, longitude))) @overload - def render_calibration( - self, - sensor: Sensor, - calibration: CalibratedSensor, - resolution: Vector2Like | None = None, - ) -> None: + def render_calibration(self, sensor: Sensor) -> None: """Render a sensor calibration. Args: - sensor (Sensor): `Sensor` object. - calibration (CalibratedSensor): `CalibratedSensor` object. - resolution (Vector2Like | None, optional): Camera resolution (width, height). + sensor (Sensor): `Sensor` record, with its extrinsics and, for a camera, its + intrinsics and image size. """ pass @@ -585,24 +579,20 @@ def render_calibration( @_check_spatial3d def render_calibration(self, *args, **kwargs) -> None: """Render a sensor calibration.""" - if len(args) + len(kwargs) <= 3: + if len(args) + len(kwargs) == 1: self._render_calibration_with_schema(*args, **kwargs) else: self._render_calibration_without_schema(*args, **kwargs) - def _render_calibration_with_schema( - self, - sensor: Sensor, - calibration: CalibratedSensor, - resolution: Vector2Like | None = None, - ) -> None: + def _render_calibration_with_schema(self, sensor: Sensor) -> None: + camera = sensor.camera self._render_calibration_without_schema( channel=sensor.channel, modality=sensor.modality, - translation=calibration.translation, - rotation=calibration.rotation, - camera_intrinsic=calibration.camera_intrinsic, - resolution=resolution, + translation=sensor.translation, + rotation=sensor.rotation, + camera_intrinsic=None if camera is None else camera.intrinsic, + resolution=None if camera is None else (camera.width, camera.height), ) def _render_calibration_without_schema( diff --git a/tests/cli/test_migrate.py b/tests/cli/test_migrate.py new file mode 100644 index 00000000..d917e0d5 --- /dev/null +++ b/tests/cli/test_migrate.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from t4_devkit.cli.migrate import cli + +SAMPLE_V0 = Path(__file__).parents[1] / "sample" / "t4dataset" + + +def test_t4migrate(tmp_path: Path) -> None: + """The 0.x sample migrates to // and passes sanity.""" + with pytest.warns(UserWarning, match="kept as float32"): + result = CliRunner().invoke(cli, [SAMPLE_V0.as_posix(), tmp_path.as_posix(), "-d", "3"]) + + assert result.exit_code == 0, result.output # t4migrate exits 1 if a sanity rule fails + manifest = json.loads((tmp_path / "t4dataset" / "3" / "manifest.json").read_text()) + assert (manifest["dataset_id"], manifest["version"]) == ("t4dataset", 3) + assert manifest["migrated_from_schema_version"] == "0.x" + assert {asset["path"] for asset in manifest["assets"]} >= { + "data/LIDAR_TOP/LIDAR_TOP.parquet", + "data/CAM_FRONT/CAM_FRONT.mp4", + "map/lanelet2_map.osm", + } diff --git a/tests/common/test_io.py b/tests/common/test_io.py index 85ad01a2..7830f841 100644 --- a/tests/common/test_io.py +++ b/tests/common/test_io.py @@ -1,6 +1,24 @@ +from __future__ import annotations + import tempfile +from pathlib import Path + +import numpy as np +import pyarrow.parquet as pq +import pypcd4 +import pytest +from PIL import Image -from t4_devkit.common.io import load_json, save_json +from t4_devkit.common.io import ( + load_image, + load_json, + load_pointcloud, + save_json, + save_pointcloud, + save_video, +) + +SAMPLE_PARQUET = Path(__file__).parents[1] / "sample/t4dataset_v1/data/LIDAR_TOP/LIDAR_TOP.parquet" def test_json_io() -> None: @@ -13,3 +31,130 @@ def test_json_io() -> None: ret = load_json(f.name) assert ret == data + + +def test_load_pointcloud_parquet() -> None: + """All fields come back in the dtype stored in the file; `columns` selects fields.""" + frame = load_pointcloud(SAMPLE_PARQUET, frame_index=1) + assert list(frame) == ["x", "y", "z", "intensity", "ring"] + assert frame["ring"].dtype == np.uint16 + assert len(frame["x"]) == 8 + + assert list(load_pointcloud(SAMPLE_PARQUET, frame_index=0, columns=["x", "z"])) == ["x", "z"] + + with pytest.raises(ValueError, match="frame_index"): + load_pointcloud(SAMPLE_PARQUET) + + +def test_load_pointcloud_bin(tmp_path: Path) -> None: + """A .bin file is read with the layout given in `fields`.""" + fields = [("x", "float32"), ("y", "float32"), ("z", "float32"), ("ring", "uint16")] + data = np.array([(1, 2, 3, 7), (4, 5, 6, 8)], dtype=fields) + data.tofile(tmp_path / "frame.bin") + + frame = load_pointcloud(tmp_path / "frame.bin", fields=fields) + assert frame["ring"].tolist() == [7, 8] + assert frame["z"].dtype == np.float32 + + with pytest.raises(ValueError, match="fields"): + load_pointcloud(tmp_path / "frame.bin") + + +def test_load_pointcloud_pcd(tmp_path: Path) -> None: + """A .pcd file is read with the layout in its header.""" + points = np.arange(10, dtype=np.float32).reshape(2, 5) + pypcd4.PointCloud.from_xyzir_points(points).save(tmp_path / "frame.pcd") + + frame = load_pointcloud(tmp_path / "frame.pcd") + assert list(frame) == ["x", "y", "z", "intensity", "ring"] + assert frame["ring"].dtype == np.uint16 + + +def test_load_image_file(tmp_path: Path) -> None: + path = tmp_path / "frame.png" + pixels = np.random.default_rng(0).integers(0, 256, (4, 6, 3), dtype=np.uint8) + Image.fromarray(pixels).save(path) + + assert np.array_equal(load_image(path), pixels) + + +def test_save_pointcloud(tmp_path: Path) -> None: + """Frames come back bit-exact in their dtypes, one row group each, empty frames included.""" + fields = [ + ("x", "float32"), + ("intensity", "uint8"), + ("channel", "uint16"), + ("time_stamp", "uint32"), + ] + rng = np.random.default_rng(0) + frames = [ + { + "x": rng.normal(size=n).astype(np.float32), + "intensity": rng.integers(0, 256, n).astype(np.uint8), + "channel": rng.integers(0, 128, n).astype(np.uint16), + "time_stamp": rng.integers(0, 100_000_000, n).astype(np.uint32), + } + for n in (5, 0, 3) + ] + path = tmp_path / "LIDAR_TOP.parquet" + + assert save_pointcloud(path, frames, fields) == 3 + for frame_index, expected in enumerate(frames): + frame = load_pointcloud(path, frame_index=frame_index) + assert list(frame) == [name for name, _ in fields] + for name, dtype in fields: + assert frame[name].dtype == np.dtype(dtype) + assert np.array_equal(frame[name], expected[name]) + + columns = pq.ParquetFile(path).metadata.row_group(0).to_dict()["columns"] + encodings = {column["path_in_schema"]: column["encodings"] for column in columns} + assert "BYTE_STREAM_SPLIT" in encodings["x"] + assert "RLE_DICTIONARY" in encodings["intensity"] + assert "DELTA_BINARY_PACKED" in encodings["channel"] + assert "DELTA_BINARY_PACKED" in encodings["time_stamp"] + + +@pytest.mark.parametrize( + ("dtype", "bit_depth", "max_value"), [(np.uint8, 8, 255), (np.uint16, 10, 65535)] +) +def test_save_video(tmp_path: Path, dtype, bit_depth: int, max_value: int) -> None: + """Frames read back by index, in any order, close to what was written.""" + images = [] + for i in range(3): + image = np.zeros((48, 64, 3), dtype=dtype) + image[..., 0] = np.linspace(0, max_value, 64).astype(dtype) + image[..., 1] = max_value // 4 * i + images.append(image) + path = tmp_path / "CAM_FRONT.mp4" + + video = save_video(path, iter(images), frame_rate=10) + assert video["frame_count"] == 3 and video["timescale"] == 10 + assert video["bit_depth"] == bit_depth + for frame_index in (2, 0, 1): + image = load_image(path, frame_index=frame_index) + assert image.dtype == dtype and image.shape == (48, 64, 3) + # lossy, but the green level of each frame is kept + green = image[..., 1].mean() / max_value + assert abs(green - 0.25 * frame_index) < 0.03 + + with pytest.raises(IndexError): + load_image(path, frame_index=3) + + +def test_save_pointcloud_null_column(tmp_path: Path) -> None: + """A field that is None in a frame is written null and left out when read.""" + fields = [("x", "float32"), ("label", "uint8")] + frames = [ + {"x": np.array([1.0, 2.0], dtype=np.float32), "label": np.array([0, 3], dtype=np.uint8)}, + {"x": np.array([4.0], dtype=np.float32), "label": None}, + ] + path = tmp_path / "LIDAR_TOP.parquet" + + assert save_pointcloud(path, frames, fields) == 2 + assert load_pointcloud(path, frame_index=0)["label"].tolist() == [0, 3] + assert list(load_pointcloud(path, frame_index=1)) == ["x"] + + # what the reader returns can be written back as it is + again = tmp_path / "again.parquet" + save_pointcloud(again, [load_pointcloud(path, frame_index=i) for i in range(2)], fields) + assert list(load_pointcloud(again, frame_index=1)) == ["x"] diff --git a/tests/dataclass/test_pointcloud.py b/tests/dataclass/test_pointcloud.py index 411a5785..1715664c 100644 --- a/tests/dataclass/test_pointcloud.py +++ b/tests/dataclass/test_pointcloud.py @@ -275,6 +275,26 @@ def test_lidar_pointcloud_from_file_reads_points_and_metainfo_from_bin(tmp_path: assert pointcloud.metainfo.source_tokens == ["lidar_front"] +@pytest.mark.parametrize(("num_pts_feats", "raises"), [(7, False), (8, True)]) +def test_lidar_pointcloud_from_v0_bin_layout( + tmp_path: Path, num_pts_feats: int, raises: bool +) -> None: + bin_filepath = tmp_path / "pointcloud.bin" + metainfo_filepath = tmp_path / "pointcloud.json" + scan = np.arange(2 * num_pts_feats, dtype=np.float32).reshape(2, num_pts_feats) + scan.tofile(bin_filepath) + metainfo_filepath.write_text( + json.dumps({"stamp": {"sec": 1, "nanosec": 2}, "num_pts_feats": num_pts_feats}) + ) + + if raises: + with pytest.raises(ValueError): + LidarPointCloud.from_file(str(bin_filepath), str(metainfo_filepath)) + else: + pointcloud = LidarPointCloud.from_file(str(bin_filepath), str(metainfo_filepath)) + assert np.array_equal(pointcloud.points, scan[:, :4].T) + + def test_lidar_pointcloud_from_file_reads_points_and_metainfo_from_pcd(tmp_path: Path) -> None: pcd_filepath = tmp_path / "pointcloud.pcd" metainfo_filepath = tmp_path / "pointcloud.json" @@ -418,3 +438,19 @@ def test_segmentation_pointcloud_from_file_reads_points_labels_and_metainfo_from assert np.array_equal(pointcloud.labels, labels) assert pointcloud.metainfo is not None assert pointcloud.metainfo.source_tokens == ["lidar_front"] + + +def test_lidar_pointcloud_from_parquet() -> None: + """A T4Dataset 1.x LiDAR frame is read from its row group by `frame_index`.""" + filepath = str( + Path(__file__).parents[1] / "sample/t4dataset_v1/data/LIDAR_TOP/LIDAR_TOP.parquet" + ) + frame0 = LidarPointCloud.from_file(filepath, frame_index=0) + frame1 = LidarPointCloud.from_file(filepath, frame_index=1) + + assert frame0.points.shape == (4, 8) + assert frame0.points.dtype == np.float32 + assert not np.array_equal(frame0.points, frame1.points) + + with pytest.raises(ValueError, match="frame_index"): + LidarPointCloud.from_file(filepath) diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 00000000..db963d6e --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any, Callable + +import pyarrow as pa +import pyarrow.parquet as pq + +from t4_devkit.sanity import sanity_check + +SAMPLE_ROOT = Path(__file__).parent / "sample" +SAMPLE_V0 = SAMPLE_ROOT / "t4dataset" +SAMPLE_V1 = SAMPLE_ROOT / "t4dataset_v1" + + +class SampleCopy: + """A writable copy of a sample dataset, for tests that change or break it. + + Examples: + >>> sample = SampleCopy(tmp_path) # the 1.0 sample, as `/sample` + >>> sample.edit("scene", lambda records: SampleCopy.set_field(records, 0, nbr_samples=5)) + >>> sample.reports()["REC012"].is_failed() + True + """ + + def __init__(self, tmp_path: Path, source: Path = SAMPLE_V1, name: str = "sample") -> None: + """ + Args: + tmp_path (Path): Directory to copy into. + source (Path, optional): Sample to copy, the 1.0 sample by default. + name (str, optional): Folder name; "sample" is the 1.0 sample's `dataset_id`. + """ + self.root = tmp_path / name + shutil.copytree(source, self.root) + + def read(self, table: str) -> Any: + """Return a table (or "manifest") as stored.""" + return json.loads(self._path(table).read_text()) + + def edit(self, table: str, change: Callable[[Any], Any]) -> None: + """Replace a table (or "manifest") with `change(records)`.""" + self._path(table).write_text(json.dumps(change(self.read(table)))) + + def reports(self, **kwargs: Any) -> dict[str, Any]: + """Return the sanity reports by rule ID; `kwargs` go to `sanity_check`.""" + result = sanity_check(self.root.as_posix(), **kwargs) + return {report.id: report for report in result.reports} + + def write_labels(self, frames: list[list[int | None] | None]) -> None: + """Add a `label` column to the 1.0 sample's LiDAR file and to its `sensor.json` fields. + + Args: + frames (list[list[int | None] | None]): Labels per frame; None for a frame without + labels. + """ + path = self.root / "data/LIDAR_TOP/LIDAR_TOP.parquet" + parquet = pq.ParquetFile(path) + schema = parquet.schema_arrow.append(pa.field("label", pa.uint8())) + with pq.ParquetWriter(path.with_suffix(".tmp"), schema) as writer: + for i, labels in enumerate(frames): + table = parquet.read_row_group(i) + column = ( + pa.nulls(table.num_rows, pa.uint8()) + if labels is None + else pa.array(labels, pa.uint8()) + ) + writer.write_table(table.append_column("label", column)) + path.with_suffix(".tmp").replace(path) + + def declare(records: list[dict]) -> list[dict]: + records[1]["lidar"]["fields"].append({"name": "label", "dtype": "uint8"}) + return records + + self.edit("sensor", declare) + + @staticmethod + def set_field(records: list[dict], position: int, **fields: Any) -> list[dict]: + """Return the records with `fields` set on the record at `position`.""" + records[position].update(fields) + return records + + def _path(self, table: str) -> Path: + if table == "manifest": + return self.root / "manifest.json" + return self.root / "annotation" / f"{table}.json" diff --git a/tests/sample/t4dataset_v1/annotation/attribute.json b/tests/sample/t4dataset_v1/annotation/attribute.json new file mode 100644 index 00000000..223e27b0 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/attribute.json @@ -0,0 +1,7 @@ +[ + { + "token": "attribute_moving", + "name": "vehicle_state.moving", + "description": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/caption.json b/tests/sample/t4dataset_v1/annotation/caption.json new file mode 100644 index 00000000..12a21b1a --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/caption.json @@ -0,0 +1,19 @@ +[ + { + "token": "caption0", + "log_token": "log", + "start_timestamp": 1704067200000000, + "end_timestamp": 1704067200200000, + "sample_data_token": null, + "scenario_token": "scenario0", + "instance_tokens": [ + "instance_car" + ], + "caption_type": "description", + "text": "The ego vehicle follows a car.", + "language": "en", + "annotation_type": "manual", + "provenance": null, + "confidence": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/category.json b/tests/sample/t4dataset_v1/annotation/category.json new file mode 100644 index 00000000..a5d54104 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/category.json @@ -0,0 +1,20 @@ +[ + { + "token": "category_car", + "name": "car", + "index": null, + "description": null, + "is_thing": true, + "has_orientation": false, + "has_number": false + }, + { + "token": "category_road", + "name": "road", + "index": null, + "description": null, + "is_thing": false, + "has_orientation": false, + "has_number": false + } +] diff --git a/tests/sample/t4dataset_v1/annotation/ego_pose.json b/tests/sample/t4dataset_v1/annotation/ego_pose.json new file mode 100644 index 00000000..c44fb1c9 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/ego_pose.json @@ -0,0 +1,112 @@ +[ + { + "token": "ego_cam0", + "translation": [ + 0.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "timestamp": 1704067200000000, + "twist": null, + "acceleration": null, + "imu": null, + "geocoordinate": null, + "source": "localization", + "frame": "map", + "covariance": null + }, + { + "token": "ego_cam1", + "translation": [ + 1.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "timestamp": 1704067200100000, + "twist": null, + "acceleration": null, + "imu": null, + "geocoordinate": null, + "source": "localization", + "frame": "map", + "covariance": null + }, + { + "token": "ego_lidar0", + "translation": [ + 0.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "timestamp": 1704067200000000, + "twist": null, + "acceleration": null, + "imu": null, + "geocoordinate": null, + "source": "localization", + "frame": "map", + "covariance": null + }, + { + "token": "ego_lidar1", + "translation": [ + 1.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "timestamp": 1704067200100000, + "twist": null, + "acceleration": null, + "imu": null, + "geocoordinate": null, + "source": "localization", + "frame": "map", + "covariance": null + }, + { + "token": "ego_lidar2", + "translation": [ + 2.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "timestamp": 1704067200200000, + "twist": null, + "acceleration": null, + "imu": null, + "geocoordinate": null, + "source": "localization", + "frame": "map", + "covariance": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/instance.json b/tests/sample/t4dataset_v1/annotation/instance.json new file mode 100644 index 00000000..960049a0 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/instance.json @@ -0,0 +1,15 @@ +[ + { + "token": "instance_car", + "name": "sample::0", + "index": null, + "category_token": "category_car", + "nbr_annotations": 3, + "first_annotation_token": "box0", + "last_annotation_token": "box1", + "map_element": { + "id": null, + "type": null + } + } +] diff --git a/tests/sample/t4dataset_v1/annotation/keypoint.json b/tests/sample/t4dataset_v1/annotation/keypoint.json new file mode 100644 index 00000000..bd3f13c4 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/keypoint.json @@ -0,0 +1,20 @@ +[ + { + "token": "keypoint0", + "sample_data_token": "cam0", + "instance_token": "instance_car", + "category_tokens": [ + "category_car" + ], + "keypoints": [ + [ + 12.0, + 18.0 + ] + ], + "num_keypoints": 1, + "annotation_type": "manual", + "provenance": null, + "confidence": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/log.json b/tests/sample/t4dataset_v1/annotation/log.json new file mode 100644 index 00000000..422f750d --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/log.json @@ -0,0 +1,10 @@ +[ + { + "token": "log", + "logfile": null, + "vehicle_token": "vehicle", + "data_captured": "2024-01-01T00:00:00Z", + "location": null, + "map_token": "map" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/map.json b/tests/sample/t4dataset_v1/annotation/map.json new file mode 100644 index 00000000..28828b93 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/map.json @@ -0,0 +1,24 @@ +[ + { + "token": "map", + "name": null, + "log_tokens": [ + "log" + ], + "format": "lanelet2_osm", + "filename": "map/lanelet2_map.osm", + "sha256": "4b325cf23adf604023824cc4a3243a7aac56c690fa872b68ce58e42a4529f054", + "map_id": "sample_map", + "version_id": "sample_map_version", + "projector": { + "type": "MGRS", + "mgrs_grid": "54SUE", + "origin": { + "lat": null, + "lon": null, + "alt": null + } + }, + "source": "manual" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/object_ann.json b/tests/sample/t4dataset_v1/annotation/object_ann.json new file mode 100644 index 00000000..d8bafa4b --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/object_ann.json @@ -0,0 +1,46 @@ +[ + { + "token": "object_ann0", + "sample_data_token": "cam0", + "instance_token": "instance_car", + "category_token": "category_car", + "attribute_tokens": [], + "bbox": [ + 10.0, + 10.0, + 30.0, + 20.0 + ], + "mask": null, + "orientation": null, + "number": null, + "annotation_type": "manual", + "provenance": null, + "confidence": null + }, + { + "token": "object_ann1", + "sample_data_token": "cam0", + "instance_token": null, + "category_token": "category_road", + "attribute_tokens": [], + "bbox": [ + 0.0, + 24.0, + 64.0, + 48.0 + ], + "mask": { + "size": [ + 48, + 64 + ], + "counts": "aDBoMGgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=" + }, + "orientation": null, + "number": null, + "annotation_type": "manual", + "provenance": null, + "confidence": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/route.json b/tests/sample/t4dataset_v1/annotation/route.json new file mode 100644 index 00000000..a1b3123c --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/route.json @@ -0,0 +1,48 @@ +[ + { + "token": "route0", + "log_token": "log", + "map_token": "map", + "valid_from": 1704067200000000, + "valid_until": null, + "route_uuid": null, + "start_pose": { + "translation": [ + 0.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ] + }, + "goal_pose": { + "translation": [ + 100.0, + 0.0, + 0.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ] + }, + "segments": [ + { + "preferred_primitive_id": 1000, + "primitive_ids": [ + 1000, + 1001 + ] + } + ], + "allow_modification": false, + "source": "mission_planner", + "provenance": "own_bag" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/sample.json b/tests/sample/t4dataset_v1/annotation/sample.json new file mode 100644 index 00000000..77bfeb28 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/sample.json @@ -0,0 +1,26 @@ +[ + { + "token": "sample0", + "timestamp": 1704067200000000, + "scene_token": "scene", + "has_key_frame": true, + "next": "sample1", + "prev": null + }, + { + "token": "sample1", + "timestamp": 1704067200100000, + "scene_token": "scene", + "has_key_frame": true, + "next": "sample2", + "prev": "sample0" + }, + { + "token": "sample2", + "timestamp": 1704067200200000, + "scene_token": "scene", + "has_key_frame": false, + "next": null, + "prev": "sample1" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/sample_annotation.json b/tests/sample/t4dataset_v1/annotation/sample_annotation.json new file mode 100644 index 00000000..761b4ca5 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/sample_annotation.json @@ -0,0 +1,78 @@ +[ + { + "token": "box0", + "sample_token": "sample0", + "instance_token": "instance_car", + "attribute_tokens": [ + "attribute_moving" + ], + "visibility_token": "visibility_full", + "translation": [ + 10.0, + 0.0, + 1.0 + ], + "size": [ + 1.9, + 4.5, + 1.6 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "velocity": [ + 1.0, + 0.0, + 0.0 + ], + "acceleration": null, + "num_lidar_pts": 0, + "num_radar_pts": null, + "next": "box1", + "prev": null, + "annotation_type": "manual", + "provenance": null, + "confidence": null + }, + { + "token": "box1", + "sample_token": "sample1", + "instance_token": "instance_car", + "attribute_tokens": [ + "attribute_moving" + ], + "visibility_token": "visibility_full", + "translation": [ + 11.0, + 0.0, + 1.0 + ], + "size": [ + 1.9, + 4.5, + 1.6 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "velocity": [ + 1.0, + 0.0, + 0.0 + ], + "acceleration": null, + "num_lidar_pts": 0, + "num_radar_pts": null, + "next": null, + "prev": "box0", + "annotation_type": "interpolated", + "provenance": "t4-devkit interpolation", + "confidence": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/sample_data.json b/tests/sample/t4dataset_v1/annotation/sample_data.json new file mode 100644 index 00000000..6af82e0f --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/sample_data.json @@ -0,0 +1,77 @@ +[ + { + "token": "cam0", + "sample_token": "sample0", + "ego_pose_token": "ego_cam0", + "sensor_token": "sensor_cam", + "filename": "data/CAM_FRONT/CAM_FRONT.mp4", + "frame_index": 0, + "fileformat": "av1_mp4", + "num_points": null, + "timestamp": 1704067200000000, + "is_key_frame": true, + "is_valid": true, + "next": "cam1", + "prev": null + }, + { + "token": "cam1", + "sample_token": "sample1", + "ego_pose_token": "ego_cam1", + "sensor_token": "sensor_cam", + "filename": "data/CAM_FRONT/CAM_FRONT.mp4", + "frame_index": 1, + "fileformat": "av1_mp4", + "num_points": null, + "timestamp": 1704067200100000, + "is_key_frame": true, + "is_valid": true, + "next": null, + "prev": "cam0" + }, + { + "token": "lidar0", + "sample_token": "sample0", + "ego_pose_token": "ego_lidar0", + "sensor_token": "sensor_lidar", + "filename": "data/LIDAR_TOP/LIDAR_TOP.parquet", + "frame_index": 0, + "fileformat": "parquet", + "num_points": 8, + "timestamp": 1704067200000000, + "is_key_frame": true, + "is_valid": true, + "next": "lidar1", + "prev": null + }, + { + "token": "lidar1", + "sample_token": "sample1", + "ego_pose_token": "ego_lidar1", + "sensor_token": "sensor_lidar", + "filename": "data/LIDAR_TOP/LIDAR_TOP.parquet", + "frame_index": 1, + "fileformat": "parquet", + "num_points": 8, + "timestamp": 1704067200100000, + "is_key_frame": true, + "is_valid": true, + "next": "lidar2", + "prev": "lidar0" + }, + { + "token": "lidar2", + "sample_token": "sample2", + "ego_pose_token": "ego_lidar2", + "sensor_token": "sensor_lidar", + "filename": "data/LIDAR_TOP/LIDAR_TOP.parquet", + "frame_index": 2, + "fileformat": "parquet", + "num_points": 8, + "timestamp": 1704067200200000, + "is_key_frame": false, + "is_valid": true, + "next": null, + "prev": "lidar1" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/scenario.json b/tests/sample/t4dataset_v1/annotation/scenario.json new file mode 100644 index 00000000..807ba4dd --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/scenario.json @@ -0,0 +1,26 @@ +[ + { + "token": "scenario0", + "log_token": "log", + "start_timestamp": 1704067200000000, + "end_timestamp": 1704067200300000, + "key_timestamp": null, + "status": null, + "lateral_decision": null, + "longitudinal_decision": null, + "events": [ + "following" + ], + "scenery": null, + "dynamic_entities": null, + "justification": null, + "taxonomy": { + "id": "cod-scene", + "version": "0.1.1" + }, + "annotation_type": "auto", + "provenance": "sample classifier", + "confidence": 0.9, + "raw": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/scene.json b/tests/sample/t4dataset_v1/annotation/scene.json new file mode 100644 index 00000000..dfb42337 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/scene.json @@ -0,0 +1,11 @@ +[ + { + "token": "scene", + "name": "sample_scene", + "description": null, + "log_token": "log", + "nbr_samples": 3, + "first_sample_token": "sample0", + "last_sample_token": "sample2" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/sensor.json b/tests/sample/t4dataset_v1/annotation/sensor.json new file mode 100644 index 00000000..c8524b89 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/sensor.json @@ -0,0 +1,124 @@ +[ + { + "token": "sensor_cam", + "channel": "CAM_FRONT", + "index": 0, + "modality": "camera", + "translation": [ + 1.0, + 0.0, + 1.5 + ], + "rotation": [ + 0.5, + -0.5, + 0.5, + -0.5 + ], + "camera": { + "width": 64, + "height": 48, + "intrinsic": [ + [ + 50.0, + 0.0, + 32.0 + ], + [ + 0.0, + 50.0, + 24.0 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "distortion": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "distortion_model": "plumb_bob", + "is_rectified": false, + "video": { + "path": "data/CAM_FRONT/CAM_FRONT.mp4", + "format": "av1_mp4", + "codec_profile": "svt-av1 preset6 crf26 intra", + "pixel_format": "yuv420p full-range bt601", + "bit_depth": 8, + "frame_count": 2, + "timescale": 10 + } + }, + "lidar": null, + "radar": null, + "hardware": { + "make": "Sony", + "model": "IMX728", + "serial_number": null, + "firmware_version": null + } + }, + { + "token": "sensor_lidar", + "channel": "LIDAR_TOP", + "index": 1, + "modality": "lidar", + "translation": [ + 0.0, + 0.0, + 2.0 + ], + "rotation": [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + "camera": null, + "lidar": { + "container": { + "format": "parquet", + "path": "data/LIDAR_TOP/LIDAR_TOP.parquet", + "frame_count": 3, + "compression": "zstd" + }, + "fields": [ + { + "name": "x", + "dtype": "float32" + }, + { + "name": "y", + "dtype": "float32" + }, + { + "name": "z", + "dtype": "float32" + }, + { + "name": "intensity", + "dtype": "float32" + }, + { + "name": "ring", + "dtype": "uint16" + } + ], + "source_channels": [], + "frame_id": "base_link", + "is_undistorted": true + }, + "radar": null, + "hardware": { + "make": "Hesai", + "model": "OT128", + "serial_number": null, + "firmware_version": null + } + } +] diff --git a/tests/sample/t4dataset_v1/annotation/traffic_light.json b/tests/sample/t4dataset_v1/annotation/traffic_light.json new file mode 100644 index 00000000..eacec538 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/traffic_light.json @@ -0,0 +1,20 @@ +[ + { + "token": "traffic_light0", + "timestamp": 1704067200000000, + "regulatory_element_id": 2000, + "elements": [ + { + "color": "green", + "shape": "circle", + "status": "solid_on", + "confidence": 1.0 + } + ], + "object_ann_token": null, + "instance_token": null, + "annotation_type": "auto", + "provenance": "sample recognizer", + "confidence": 0.9 + } +] diff --git a/tests/sample/t4dataset_v1/annotation/vehicle.json b/tests/sample/t4dataset_v1/annotation/vehicle.json new file mode 100644 index 00000000..c31edf89 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/vehicle.json @@ -0,0 +1,13 @@ +[ + { + "token": "vehicle", + "name": null, + "model": "sample_vehicle", + "length": 4.9, + "width": 1.9, + "height": 2.0, + "wheel_base": 2.8, + "rear_axle_to_center": null, + "sensor_rig_id": null + } +] diff --git a/tests/sample/t4dataset_v1/annotation/vehicle_state.json b/tests/sample/t4dataset_v1/annotation/vehicle_state.json new file mode 100644 index 00000000..7b5b4fe1 --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/vehicle_state.json @@ -0,0 +1,21 @@ +[ + { + "token": "vehicle_state0", + "timestamp": 1704067200000000, + "accel_pedal": 0.0, + "brake_pedal": 0.0, + "steer_pedal": null, + "steering_tire_angle": 0.0, + "steering_wheel_angle": null, + "shift_state": "DRIVE", + "indicators": { + "left": "off", + "right": "off", + "hazard": "off" + }, + "longitudinal_velocity": 1.0, + "lateral_velocity": 0.0, + "heading_rate": 0.0, + "source": "vehicle_interface" + } +] diff --git a/tests/sample/t4dataset_v1/annotation/visibility.json b/tests/sample/t4dataset_v1/annotation/visibility.json new file mode 100644 index 00000000..0c12fd8e --- /dev/null +++ b/tests/sample/t4dataset_v1/annotation/visibility.json @@ -0,0 +1,7 @@ +[ + { + "token": "visibility_full", + "level": "full", + "description": null + } +] diff --git a/tests/sample/t4dataset_v1/data/CAM_FRONT/CAM_FRONT.mp4 b/tests/sample/t4dataset_v1/data/CAM_FRONT/CAM_FRONT.mp4 new file mode 100644 index 00000000..3e49e0c8 Binary files /dev/null and b/tests/sample/t4dataset_v1/data/CAM_FRONT/CAM_FRONT.mp4 differ diff --git a/tests/sample/t4dataset_v1/data/LIDAR_TOP/LIDAR_TOP.parquet b/tests/sample/t4dataset_v1/data/LIDAR_TOP/LIDAR_TOP.parquet new file mode 100644 index 00000000..37928b6f Binary files /dev/null and b/tests/sample/t4dataset_v1/data/LIDAR_TOP/LIDAR_TOP.parquet differ diff --git a/tests/sample/t4dataset_v1/make_sample_v1.py b/tests/sample/t4dataset_v1/make_sample_v1.py new file mode 100644 index 00000000..c5062359 --- /dev/null +++ b/tests/sample/t4dataset_v1/make_sample_v1.py @@ -0,0 +1,525 @@ +"""Generate the T4Dataset 1.0 sample in this directory. + +Three samples (frames): two key frames with one camera (CAM_FRONT, AV1) and one LiDAR +(LIDAR_TOP, Parquet) frame each, then a non-key frame with a LiDAR frame only (a channel can be +missing). One record in every optional table. All data is synthetic and deterministic. + +Written with `DatasetWriter`, like a real dataset (no FFmpeg needed): + + python tests/sample/t4dataset_v1/make_sample_v1.py +""" + +from __future__ import annotations + +import base64 +import hashlib +import shutil +import tempfile +from pathlib import Path + +import numpy as np +from pycocotools import mask as cocomask + +from t4_devkit.schema.v1 import SCHEMAS, DatasetWriter, SchemaName + +ROOT = Path(__file__).parent + +NUM_FRAMES = 2 # key frames, with a camera and a LiDAR frame each +NUM_SAMPLES = NUM_FRAMES + 1 # the last sample is a non-key frame with a LiDAR frame only +FPS = 10 +WIDTH, HEIGHT = 64, 48 +NUM_POINTS = 8 +T0 = 1704067200000000 # 2024-01-01T00:00:00Z in microseconds +DT = 1_000_000 // FPS + +LIDAR_FIELDS = [ + ("x", "float32"), + ("y", "float32"), + ("z", "float32"), + ("intensity", "float32"), + ("ring", "uint16"), +] +MAP_PATH = "map/lanelet2_map.osm" + + +def road_mask() -> dict[str, object]: + """RLE mask of the road: the bottom half of the image (rows 24-47), as pycocotools writes it.""" + mask = np.zeros((HEIGHT, WIDTH), dtype=np.uint8, order="F") + mask[24:, :] = 1 + rle = cocomask.encode(mask) + return {"size": [HEIGHT, WIDTH], "counts": base64.b64encode(rle["counts"]).decode()} + + +def camera_frames() -> list[np.ndarray]: + """NUM_FRAMES synthetic RGB frames.""" + frames = [] + for i in range(NUM_FRAMES): + frame = np.zeros((HEIGHT, WIDTH, 3), dtype=np.uint8) + frame[:, :, 0] = np.linspace(0, 255, WIDTH, dtype=np.uint8) + frame[:, :, 1] = 128 * i + frames.append(frame) + return frames + + +def lidar_frames() -> list[dict[str, np.ndarray]]: + """NUM_SAMPLES frames of NUM_POINTS points.""" + rng = np.random.default_rng(0) + frames = [] + for _ in range(NUM_SAMPLES): + xyzi = rng.uniform(-10, 10, (NUM_POINTS, 4)).astype(np.float32) + frame = {name: xyzi[:, k] for k, (name, _) in enumerate(LIDAR_FIELDS[:4])} + frame["ring"] = np.arange(NUM_POINTS, dtype=np.uint16) + frames.append(frame) + return frames + + +# Added to the 0.x sample's map: a traffic light (way) and its regulatory element, controlling +# the main road lanelet 1000. +TRAFFIC_LIGHT_WAY = 200 +TRAFFIC_LIGHT_REGULATORY_ELEMENT = 2000 +LANELETS = [1000, 1001] +MAP_ADDITIONS = f""" + + + + + + + + + + + + + + +""" + + +def write_map(path: Path) -> None: + """Write the 0.x sample's map with the traffic light added.""" + osm = (ROOT.parent / "t4dataset" / MAP_PATH).read_text() + path.write_text(osm.replace("\n\n", MAP_ADDITIONS)) + + +def label( + annotation_type: str = "manual", + provenance: str | None = None, + confidence: float | None = None, +) -> dict: + return {"annotation_type": annotation_type, "provenance": provenance, "confidence": confidence} + + +def tables(camera_video: dict, lidar_path: str, map_sha256: str) -> dict[str, list[dict]]: + identity = [1.0, 0.0, 0.0, 0.0] + zero = [0.0, 0.0, 0.0] + samples = [f"sample{i}" for i in range(NUM_SAMPLES)] + timestamps = [T0 + i * DT for i in range(NUM_SAMPLES)] + + def link(tokens: list[str], i: int) -> dict: + return { + "next": tokens[i + 1] if i + 1 < len(tokens) else None, + "prev": tokens[i - 1] if i > 0 else None, + } + + sample_data = [] + ego_pose = [] + for channel, sensor, path, fileformat, num_points, num_frames in ( + ("cam", "sensor_cam", camera_video["path"], "av1_mp4", None, NUM_FRAMES), + ("lidar", "sensor_lidar", lidar_path, "parquet", NUM_POINTS, NUM_SAMPLES), + ): + tokens = [f"{channel}{i}" for i in range(num_frames)] + for i, token in enumerate(tokens): + sample_data.append( + { + "token": token, + "sample_token": samples[i], + "ego_pose_token": f"ego_{token}", + "sensor_token": sensor, + "filename": path, + "frame_index": i, + "fileformat": fileformat, + "num_points": num_points, + "timestamp": timestamps[i], + "is_key_frame": i < NUM_FRAMES, + "is_valid": True, + } + | link(tokens, i) + ) + ego_pose.append( + { + "token": f"ego_{token}", + "translation": [float(i), 0.0, 0.0], + "rotation": identity, + "timestamp": timestamps[i], + "twist": None, + "acceleration": None, + "imu": None, + "geocoordinate": None, + "source": "localization", + "frame": "map", + "covariance": None, + } + ) + + sample_annotations = [f"box{i}" for i in range(NUM_FRAMES)] + return { + "sample": [ + { + "token": s, + "timestamp": timestamps[i], + "scene_token": "scene", + "has_key_frame": i < NUM_FRAMES, + } + | link(samples, i) + for i, s in enumerate(samples) + ], + "sample_data": sample_data, + "ego_pose": ego_pose, + "sensor": [ + { + "token": "sensor_cam", + "channel": "CAM_FRONT", + "index": 0, + "modality": "camera", + "translation": [1.0, 0.0, 1.5], + "rotation": [0.5, -0.5, 0.5, -0.5], + "camera": { + "width": WIDTH, + "height": HEIGHT, + "intrinsic": [[50.0, 0.0, 32.0], [0.0, 50.0, 24.0], [0.0, 0.0, 1.0]], + "distortion": [0.0, 0.0, 0.0, 0.0, 0.0], + "distortion_model": "plumb_bob", + "is_rectified": False, + "video": camera_video, + }, + "lidar": None, + "radar": None, + "hardware": { + "make": "Sony", + "model": "IMX728", + "serial_number": None, + "firmware_version": None, + }, + }, + { + "token": "sensor_lidar", + "channel": "LIDAR_TOP", + "index": 1, + "modality": "lidar", + "translation": [0.0, 0.0, 2.0], + "rotation": identity, + "camera": None, + "lidar": { + "container": { + "format": "parquet", + "path": lidar_path, + "frame_count": NUM_SAMPLES, + "compression": "zstd", + }, + "fields": [{"name": name, "dtype": dtype} for name, dtype in LIDAR_FIELDS], + "source_channels": [], + "frame_id": "base_link", + "is_undistorted": True, + }, + "radar": None, + "hardware": { + "make": "Hesai", + "model": "OT128", + "serial_number": None, + "firmware_version": None, + }, + }, + ], + "scene": [ + { + "token": "scene", + "name": "sample_scene", + "description": None, + "log_token": "log", + "nbr_samples": NUM_SAMPLES, + "first_sample_token": samples[0], + "last_sample_token": samples[-1], + } + ], + "log": [ + { + "token": "log", + "logfile": None, + "vehicle_token": "vehicle", + "data_captured": "2024-01-01T00:00:00Z", + "location": None, + "map_token": "map", + } + ], + "vehicle": [ + { + "token": "vehicle", + "name": None, + "model": "sample_vehicle", + "length": 4.9, + "width": 1.9, + "height": 2.0, + "wheel_base": 2.8, + "rear_axle_to_center": None, + "sensor_rig_id": None, + } + ], + "map": [ + { + "token": "map", + "name": None, + "log_tokens": ["log"], + "format": "lanelet2_osm", + "filename": MAP_PATH, + "sha256": map_sha256, + "map_id": "sample_map", + "version_id": "sample_map_version", + "projector": { + "type": "MGRS", + "mgrs_grid": "54SUE", + "origin": {"lat": None, "lon": None, "alt": None}, + }, + "source": "manual", + } + ], + "vehicle_state": [ + { + "token": "vehicle_state0", + "timestamp": timestamps[0], + "accel_pedal": 0.0, + "brake_pedal": 0.0, + "steer_pedal": None, + "steering_tire_angle": 0.0, + "steering_wheel_angle": None, + "shift_state": "DRIVE", + "indicators": {"left": "off", "right": "off", "hazard": "off"}, + "longitudinal_velocity": 1.0, + "lateral_velocity": 0.0, + "heading_rate": 0.0, + "source": "vehicle_interface", + } + ], + "category": [ + { + "token": "category_car", + "name": "car", + "index": None, + "description": None, + "is_thing": True, + "has_orientation": False, + "has_number": False, + }, + { + "token": "category_road", + "name": "road", + "index": None, + "description": None, + "is_thing": False, + "has_orientation": False, + "has_number": False, + }, + ], + "attribute": [ + {"token": "attribute_moving", "name": "vehicle_state.moving", "description": None} + ], + "visibility": [{"token": "visibility_full", "level": "full", "description": None}], + "instance": [ + { + "token": "instance_car", + "name": "sample::0", + "index": None, + "category_token": "category_car", + # its 3D boxes and its 2D label + "nbr_annotations": NUM_FRAMES + 1, + "first_annotation_token": sample_annotations[0], + "last_annotation_token": sample_annotations[-1], + "map_element": {"id": None, "type": None}, + } + ], + "sample_annotation": [ + { + "token": token, + "sample_token": samples[i], + "instance_token": "instance_car", + "attribute_tokens": ["attribute_moving"], + "visibility_token": "visibility_full", + "translation": [10.0 + i, 0.0, 1.0], + "size": [1.9, 4.5, 1.6], + "rotation": identity, + "velocity": [1.0, 0.0, 0.0], + "acceleration": None, + "num_lidar_pts": 0, + "num_radar_pts": None, + } + | link(sample_annotations, i) + | (label() if i == 0 else label("interpolated", provenance="t4-devkit interpolation")) + for i, token in enumerate(sample_annotations) + ], + "object_ann": [ + { + "token": "object_ann0", + "sample_data_token": "cam0", + "instance_token": "instance_car", + "category_token": "category_car", + "attribute_tokens": [], + "bbox": [10.0, 10.0, 30.0, 20.0], + "mask": None, + "orientation": None, + "number": None, + } + | label(), + { + "token": "object_ann1", + "sample_data_token": "cam0", + "instance_token": None, + "category_token": "category_road", + "attribute_tokens": [], + "bbox": [0.0, 24.0, float(WIDTH), float(HEIGHT)], + "mask": road_mask(), + "orientation": None, + "number": None, + } + | label(), + ], + "keypoint": [ + { + "token": "keypoint0", + "sample_data_token": "cam0", + "instance_token": "instance_car", + "category_tokens": ["category_car"], + "keypoints": [[12.0, 18.0]], + "num_keypoints": 1, + } + | label() + ], + "traffic_light": [ + { + "token": "traffic_light0", + "timestamp": timestamps[0], + "regulatory_element_id": TRAFFIC_LIGHT_REGULATORY_ELEMENT, + "elements": [ + {"color": "green", "shape": "circle", "status": "solid_on", "confidence": 1.0} + ], + "object_ann_token": None, + "instance_token": None, + } + | label("auto", provenance="sample recognizer", confidence=0.9) + ], + "route": [ + { + "token": "route0", + "log_token": "log", + "map_token": "map", + "valid_from": timestamps[0], + "valid_until": None, + "route_uuid": None, + "start_pose": {"translation": zero, "rotation": identity}, + "goal_pose": {"translation": [100.0, 0.0, 0.0], "rotation": identity}, + "segments": [{"preferred_primitive_id": LANELETS[0], "primitive_ids": LANELETS}], + "allow_modification": False, + "source": "mission_planner", + "provenance": "own_bag", + } + ], + "scenario": [ + { + "token": "scenario0", + "log_token": "log", + "start_timestamp": timestamps[0], + "end_timestamp": timestamps[-1] + DT, + "key_timestamp": None, + "status": None, + "lateral_decision": None, + "longitudinal_decision": None, + "events": ["following"], + "scenery": None, + "dynamic_entities": None, + "justification": None, + "taxonomy": {"id": "cod-scene", "version": "0.1.1"}, + "raw": None, + } + | label("auto", provenance="sample classifier", confidence=0.9) + ], + "caption": [ + { + "token": "caption0", + "log_token": "log", + "start_timestamp": timestamps[0], + "end_timestamp": timestamps[-1], + "sample_data_token": None, + "scenario_token": "scenario0", + "instance_tokens": ["instance_car"], + "caption_type": "description", + "text": "The ego vehicle follows a car.", + "language": "en", + } + | label() + ], + } + + +# Manifest fields the writer does not compute (it adds the files, capabilities and t4-devkit). +MANIFEST_INFO = { + "schema_version": "1.0.0", + "dataset_id": "sample", + "version": 0, + "created_at": "2024-01-01T00:00:00Z", + "generators": [], + "annotation_specifications": {}, + "provenance": { + "rosbag": { + "log_file_ids": [], + "storage": None, + "record_start": "2024-01-01T00:00:00Z", + "record_end": "2024-01-01T00:00:01Z", + }, + "topics": { + "CAM_FRONT": "/sensing/camera/camera0/image_rect_color/compressed", + "LIDAR_TOP": "/sensing/lidar/concatenated/pointcloud", + }, + "process_config_id": None, + "process_config_version_id": None, + }, + "project_id": None, + "source_project_id": None, + "is_synthetic": True, + "is_discontinuous": None, + "migrated_from_schema_version": None, + "migration_notes": None, +} + + +def write(output_root: Path) -> None: + """Write the sample dataset to `output_root`, which must not exist.""" + writer = DatasetWriter(output_root) + camera_video = writer.write_camera("CAM_FRONT", camera_frames(), FPS) + lidar_path, _ = writer.write_lidar("LIDAR_TOP", lidar_frames(), LIDAR_FIELDS) + with tempfile.TemporaryDirectory() as tmp: + map_file = Path(tmp) / "lanelet2_map.osm" + write_map(map_file) + writer.copy(map_file, MAP_PATH) + map_sha256 = hashlib.sha256((output_root / MAP_PATH).read_bytes()).hexdigest() + records = { + SchemaName(name): [SCHEMAS.get(SchemaName(name)).from_dict(data) for data in rows] + for name, rows in tables(camera_video, lidar_path, map_sha256).items() + } + writer.save(records, MANIFEST_INFO) + + +def main() -> None: + """Write the sample dataset in place of this directory's.""" + with tempfile.TemporaryDirectory() as tmp: + output_root = Path(tmp) / "sample" + write(output_root) + for name in ("annotation", "data", "map", "manifest.json"): + target = ROOT / name + if target.is_dir(): + shutil.rmtree(target) + elif target.exists(): + target.unlink() + shutil.move(output_root / name, target) + + +if __name__ == "__main__": + main() diff --git a/tests/sample/t4dataset_v1/manifest.json b/tests/sample/t4dataset_v1/manifest.json new file mode 100644 index 00000000..1324412d --- /dev/null +++ b/tests/sample/t4dataset_v1/manifest.json @@ -0,0 +1,156 @@ +{ + "schema_version": "1.0.0", + "dataset_id": "sample", + "version": 0, + "created_at": "2024-01-01T00:00:00Z", + "generators": [ + { + "name": "t4-devkit", + "version": "0.8.0" + } + ], + "capabilities": { + "tasks": [ + "detection3d", + "detection2d", + "segmentation2d", + "keypoint", + "traffic_light", + "planning" + ], + "modalities": { + "camera": [ + "CAM_FRONT" + ], + "lidar": [ + "LIDAR_TOP" + ], + "radar": [] + }, + "map": true, + "route": true, + "traffic_light": true, + "vehicle_state": true, + "scenario": true + }, + "annotation_specifications": {}, + "tables": { + "sample": { + "path": "annotation/sample.json", + "sha256": "beede65ca85b04597d5b29a71d48df9d983daa75778f9ffb6bbd6b03c838d45c" + }, + "sample_data": { + "path": "annotation/sample_data.json", + "sha256": "cb66c0f4bd72a2e08452ba79f6424887c199d53d2c611830640ca4961271694c" + }, + "ego_pose": { + "path": "annotation/ego_pose.json", + "sha256": "c797549026aa60e7e5aa5578511754d62de75c52fc9f079fd7c914796a1f6c79" + }, + "sensor": { + "path": "annotation/sensor.json", + "sha256": "dd66fc0cadb8cb1e54831067a7f13c9f159de8d686f5f9ba85976706e3ecc80e" + }, + "scene": { + "path": "annotation/scene.json", + "sha256": "8b6223870fbfa4c49cc771c8f79f8acfbd1656a54146b0b8e4814384304989f5" + }, + "log": { + "path": "annotation/log.json", + "sha256": "0c7a005878e85e1888fca09d437c9fbfb00c363fc688d406569560520e776cd8" + }, + "vehicle": { + "path": "annotation/vehicle.json", + "sha256": "c86da3382dbaa41d0e3bd9cd0f0e11469b4adf7031a787f49da0e97b12a0c9cb" + }, + "map": { + "path": "annotation/map.json", + "sha256": "5cd6ff58a1c1d942de187f2902239091e2f662d21d1d564975b300a7c9d4fbab" + }, + "vehicle_state": { + "path": "annotation/vehicle_state.json", + "sha256": "4a82c0637a5aae34d44f3a5aa056953e6da500e322372c4f12b9a5368e7090ab" + }, + "category": { + "path": "annotation/category.json", + "sha256": "aa251dcf936e13aad1aff82eda3fc400b3b3616ad88134564bed56fac6cb3320" + }, + "attribute": { + "path": "annotation/attribute.json", + "sha256": "6691c3ce8d6dee39c1771962558548c5348b2decc0896765016c8a44e4568dbf" + }, + "visibility": { + "path": "annotation/visibility.json", + "sha256": "3e33c164c70bd5262ec8e667606de935f25613059df5eed94a41e9519f8c0811" + }, + "instance": { + "path": "annotation/instance.json", + "sha256": "c70b532929453d1f759eaa41a666e4145efb7d97a41f7cc3758dae3668f20fa3" + }, + "sample_annotation": { + "path": "annotation/sample_annotation.json", + "sha256": "85afcc5f772d398bafc56b77df9c726d8c0a0e3e1e4421c666bf527cb4adedf8" + }, + "object_ann": { + "path": "annotation/object_ann.json", + "sha256": "5434655b3ce0246de3418e79760c97b428ed5ad2c89964abcc67e9a52ec2873b" + }, + "keypoint": { + "path": "annotation/keypoint.json", + "sha256": "1aff568f5509d4b44a297c80b56aea77c6d6264d3e7992b8dff741a1b59ecac0" + }, + "traffic_light": { + "path": "annotation/traffic_light.json", + "sha256": "989e24ff88fe14aaf6da6d524b6dbfea7e96fd36a67407cf8f16193674a9239d" + }, + "route": { + "path": "annotation/route.json", + "sha256": "80b6d0d982fc7d7a972a76e14f348421d7586ba84590f007836857eaa071abc7" + }, + "scenario": { + "path": "annotation/scenario.json", + "sha256": "8c5b20607c45a4e2cf92ff3b291a036550ed1c0b9d1534f4fa2f4c8dc978c8ad" + }, + "caption": { + "path": "annotation/caption.json", + "sha256": "377049baba19c58c7dbb376d67dc6c122fc7aef262be7e81eb04e7c4b1e05e00" + } + }, + "assets": [ + { + "path": "data/CAM_FRONT/CAM_FRONT.mp4", + "sha256": "2c98ef16ed7269de658b46d5ece3698a9ba3f76449c6487e1c737c1e7fd00a93", + "bytes": 939 + }, + { + "path": "data/LIDAR_TOP/LIDAR_TOP.parquet", + "sha256": "ea286e166f6a4e5266e2765ff60e84f27eef1d32517621ea7bf4f56ee2a2b2f8", + "bytes": 3045 + }, + { + "path": "map/lanelet2_map.osm", + "sha256": "4b325cf23adf604023824cc4a3243a7aac56c690fa872b68ce58e42a4529f054", + "bytes": 6055 + } + ], + "provenance": { + "rosbag": { + "log_file_ids": [], + "storage": null, + "record_start": "2024-01-01T00:00:00Z", + "record_end": "2024-01-01T00:00:01Z" + }, + "topics": { + "CAM_FRONT": "/sensing/camera/camera0/image_rect_color/compressed", + "LIDAR_TOP": "/sensing/lidar/concatenated/pointcloud" + }, + "process_config_id": null, + "process_config_version_id": null + }, + "project_id": null, + "source_project_id": null, + "is_synthetic": true, + "is_discontinuous": null, + "migrated_from_schema_version": null, + "migration_notes": null +} diff --git a/tests/sample/t4dataset_v1/map/lanelet2_map.osm b/tests/sample/t4dataset_v1/map/lanelet2_map.osm new file mode 100644 index 00000000..98ea03d3 --- /dev/null +++ b/tests/sample/t4dataset_v1/map/lanelet2_map.osm @@ -0,0 +1,208 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/sample/t4dataset_v1/test_sample_v1.py b/tests/sample/t4dataset_v1/test_sample_v1.py new file mode 100644 index 00000000..736d9ba7 --- /dev/null +++ b/tests/sample/t4dataset_v1/test_sample_v1.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from pathlib import Path + +import make_sample_v1 + +SAMPLE_V1 = Path(__file__).parent + + +def test_generator_reproduces_sample(tmp_path: Path) -> None: + """`make_sample_v1.py` writes the committed sample: run it after changing either. + + The video and the manifest (its hash and the t4-devkit version) are left out: their bytes + depend on the encoder build and the installed version. + """ + make_sample_v1.write(tmp_path / "sample") + paths = [ + *(f"annotation/{path.name}" for path in (SAMPLE_V1 / "annotation").glob("*.json")), + "data/LIDAR_TOP/LIDAR_TOP.parquet", + "map/lanelet2_map.osm", + ] + for path in paths: + assert (tmp_path / "sample" / path).read_bytes() == (SAMPLE_V1 / path).read_bytes(), path diff --git a/tests/sanity/test_v1_checkers.py b/tests/sanity/test_v1_checkers.py new file mode 100644 index 00000000..0cafb30d --- /dev/null +++ b/tests/sanity/test_v1_checkers.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from pathlib import Path + +from t4_devkit.sanity import sanity_check +from t4_devkit.sanity.checker import SchemaVersion +from t4_devkit.sanity.registry import CHECKERS +from tests.helpers import SAMPLE_V0, SAMPLE_V1, SampleCopy + + +def test_rules_by_version() -> None: + """Each dataset gets only the rules of its version; 0.x keeps exactly its 0.x rules.""" + v0_rules = { + id + for rules in CHECKERS.values() + for id, checker in rules.items() + if SchemaVersion.V0 in checker.versions + } + assert {report.id for report in sanity_check(SAMPLE_V0.as_posix()).reports} == v0_rules + + v1_reports = {report.id for report in sanity_check(SAMPLE_V1.as_posix()).reports} + assert "STR006" not in v1_reports and "FMT001" not in v1_reports + assert {"STR001", "TIV001", "STR010", "FMT020", "FMT040"} <= v1_reports + + +def test_v1_sample_passes(tmp_path: Path) -> None: + """The 1.0 sample passes every rule, hashes included; the only warnings are the missing + version, bag and pointcloud map directories.""" + result = sanity_check(SampleCopy(tmp_path).root.as_posix(), checksum=True) + assert result.is_passed() + warnings = {report.id for report in result.reports if not report.is_passed(strict=True)} + assert warnings == {"STR001", "STR005", "STR009"} + + +def test_str010_unsupported_version(tmp_path: Path) -> None: + """An unsupported schema version fails, and the other 1.0 rules are skipped.""" + sample = SampleCopy(tmp_path) + sample.edit("manifest", lambda m: dict(m, schema_version="2.0.0")) + reports = sample.reports() + assert reports["STR010"].is_failed() + assert reports["FMT020"].is_failed() + assert reports["STR011"].is_skipped() and reports["FMT021"].is_skipped() + + +def test_str011_missing_file(tmp_path: Path) -> None: + sample = SampleCopy(tmp_path) + (sample.root / "map" / "lanelet2_map.osm").unlink() + assert sample.reports()["STR011"].is_failed() + + +def test_str012_checksum(tmp_path: Path) -> None: + """Hashes are verified only with `checksum`; a changed file then fails.""" + sample = SampleCopy(tmp_path) + with (sample.root / "map" / "lanelet2_map.osm").open("a") as f: + f.write(" ") + assert sample.reports()["STR012"].severity == "WARNING" + report = sample.reports(checksum=True)["STR012"] + assert report.is_failed() and "lanelet2_map.osm" in report.reasons[0] + + +def test_str013_unlisted_file(tmp_path: Path) -> None: + sample = SampleCopy(tmp_path) + (sample.root / "data" / "extra.txt").write_text("not listed") + report = sample.reports()["STR013"] + assert not report.is_passed(strict=True) and "data/extra.txt" in report.reasons[0] + + +def test_str014_required_table(tmp_path: Path) -> None: + """An unlisted required table fails STR014, and its format rule is skipped.""" + sample = SampleCopy(tmp_path) + sample.edit( + "manifest", + lambda m: dict(m, tables={k: v for k, v in m["tables"].items() if k != "vehicle"}), + ) + reports = sample.reports() + assert reports["STR014"].is_failed() and "vehicle" in reports["STR014"].reasons[0] + assert reports["FMT027"].is_skipped() + + +def test_str015_identity(tmp_path: Path) -> None: + report = SampleCopy(tmp_path, name="other_id").reports()["STR015"] + assert not report.is_passed(strict=True) and "'sample'" in report.reasons[0] + + +def test_fmt_invalid_record(tmp_path: Path) -> None: + """A record that does not build into its 1.0 class fails its table's format rule.""" + sample = SampleCopy(tmp_path) + token = sample.read("sensor")[0]["token"] + sample.edit("sensor", lambda records: SampleCopy.set_field(records, 0, index=300)) + report = sample.reports()["FMT024"] + assert report.is_failed() and token in report.reasons[0] + + +def test_fmt_unreadable_table(tmp_path: Path) -> None: + sample = SampleCopy(tmp_path) + (sample.root / "annotation" / "scene.json").write_text("not json") + report = sample.reports()["FMT025"] + assert report.is_failed() and "cannot be read" in report.reasons[0] diff --git a/tests/sanity/test_v1_record_checkers.py b/tests/sanity/test_v1_record_checkers.py new file mode 100644 index 00000000..f455c3c7 --- /dev/null +++ b/tests/sanity/test_v1_record_checkers.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.helpers import SampleCopy + +set_field = SampleCopy.set_field + + +def _second_instance(records: list[dict]) -> list[dict]: + return [*set_field(records, 0, index=5), dict(records[0], token="instance_other")] + + +def _declare_label_field(records: list[dict]) -> list[dict]: + records[1]["lidar"]["fields"].append({"name": "label", "dtype": "uint8"}) + return records + + +def _capabilities(change): + def edit(manifest: dict) -> dict: + change(manifest["capabilities"]) + return manifest + + return edit + + +# (rule, table, change): each change breaks only what the rule checks. +CASES = [ + ("REC008", "sensor", lambda r: []), + ("REC009", "sensor", lambda r: set_field(r, 1, index=0)), + ("REC010", "instance", _second_instance), + ("REC011", "sensor", _declare_label_field), + ("REC012", "scene", lambda r: set_field(r, 0, nbr_samples=5)), + ("REC013", "instance", lambda r: set_field(r, 0, nbr_annotations=2)), + ("REC014", "object_ann", lambda r: set_field(r, 1, instance_token="instance_car")), + ("REC015", "object_ann", lambda r: set_field(r, 0, orientation=1.0)), + ("REC016", "sample_annotation", lambda r: set_field(r, 0, confidence=0.5)), + ("REC017", "sample_annotation", lambda r: set_field(r, 1, provenance=None)), + ("REC018", "sample", lambda r: set_field(r, 0, has_key_frame=False)), + ("REC019", "sensor", lambda r: (r[0]["camera"]["video"].update(frame_count=3), r)[1]), + ("REC020", "sample_data", lambda r: set_field(r, 0, fileformat="jpg")), + ("REC021", "sample", lambda r: set_field(r, 1, timestamp=r[0]["timestamp"] - 1)), + ("REC022", "ego_pose", lambda r: set_field(r, 0, timestamp=1_704_067_200)), + ("REC023", "manifest", _capabilities(lambda c: c["modalities"]["camera"].append("CAM_BACK"))), + ("REC024", "manifest", _capabilities(lambda c: c.update(route=False))), + ("REC025", "vehicle", lambda r: set_field(r, 0, model="unknown")), + ("REC026", "route", lambda r: set_field(r, 0, valid_until=r[0]["valid_from"])), + ("REC027", "sensor", lambda r: set_field(r, 0, rotation=[2.0, 0.0, 0.0, 0.0])), + ("REC028", "sample_data", lambda r: set_field(r, 0, is_key_frame=False)), +] + + +@pytest.mark.parametrize(("rule", "table", "change"), CASES, ids=[case[0] for case in CASES]) +def test_record_rule_fails(tmp_path: Path, rule: str, table: str, change) -> None: + sample = SampleCopy(tmp_path) + sample.edit(table, change) + report = sample.reports()[rule] + assert not report.is_passed(strict=True), report + assert report.reasons + + +@pytest.mark.parametrize( + ("rule", "table", "change"), + [ + ("REC012", "scene", lambda r: set_field(r, 0, nbr_samples=5)), + ( + "REC013", + "instance", + lambda r: set_field(r, 0, nbr_annotations=1, last_annotation_token=None), + ), + ], +) +def test_record_rule_fix(tmp_path: Path, rule: str, table: str, change) -> None: + """`fix` recomputes the values and updates the table's hash in `manifest.json`.""" + sample = SampleCopy(tmp_path) + sample.edit(table, change) + assert sample.reports(fix=True)[rule].fixed + reports = sample.reports(checksum=True) + assert reports[rule].is_passed(strict=True) + assert reports["STR012"].is_passed(strict=True) + + +def test_rec029_point_labels(tmp_path: Path) -> None: + """Category 0 must be `unpainted`, and a frame's labels are all null or none null.""" + sample = SampleCopy(tmp_path / "bad") + sample.write_labels([[0] * 8, [1] * 7 + [None], None]) + sample.edit("category", lambda r: set_field(set_field(r, 0, index=1), 1, index=0)) + reasons = sample.reports()["REC029"].reasons + assert "not 'unpainted'" in reasons[0] + assert "frames [1]" in reasons[1] + + sample = SampleCopy(tmp_path / "ok") + sample.write_labels([[0] * 8, [1] * 8, None]) + sample.edit( + "category", + lambda r: set_field(set_field(r, 0, index=1), 1, index=0, name="unpainted"), + ) + assert sample.reports()["REC029"].is_passed(strict=True) diff --git a/tests/sanity/test_v1_reference_checkers.py b/tests/sanity/test_v1_reference_checkers.py new file mode 100644 index 00000000..b0c7d79c --- /dev/null +++ b/tests/sanity/test_v1_reference_checkers.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.helpers import SampleCopy + +set_field = SampleCopy.set_field + + +def _lidar(records: list[dict], **fields) -> list[dict]: + """Edit the `lidar` block of the LiDAR sensor.""" + records[1]["lidar"].update(fields) + return records + + +MISSING = "0" * 32 + +# (rule, table, change): each change breaks what the rule checks. +CASES = [ + ("REF015", "sample_data", lambda r: set_field(r, 0, sensor_token=MISSING)), + ("REF016", "log", lambda r: set_field(r, 0, vehicle_token=MISSING)), + ("REF017", "log", lambda r: set_field(r, 0, map_token=None)), + ("REF018", "sample_annotation", lambda r: set_field(r, 0, visibility_token=MISSING)), + ("REF019", "object_ann", lambda r: set_field(r, 0, category_token=MISSING)), + ("REF020", "keypoint", lambda r: set_field(r, 0, category_tokens=[MISSING])), + ("REF021", "traffic_light", lambda r: set_field(r, 0, object_ann_token=MISSING)), + ("REF022", "route", lambda r: set_field(r, 0, map_token=MISSING)), + ("REF023", "scenario", lambda r: set_field(r, 0, log_token=MISSING)), + ("REF024", "caption", lambda r: set_field(r, 0, scenario_token=MISSING)), + ("REF025", "sensor", lambda r: _lidar(r, source_channels=["CAM_FRONT"])), + ("REF107", "sample", lambda r: set_field(r, 0, next=r[2]["token"])), + ("REF108", "sample_data", lambda r: set_field(r, 0, next=r[3]["token"])), + ("REF204", "sample_data", lambda r: set_field(r, 0, filename="data/CAM_FRONT/other.mp4")), + ("REF205", "sensor", lambda r: (r[1]["lidar"]["fields"][3].update(dtype="float64"), r)[1]), + ( + "REF206", + "sample_data", + lambda r: [dict(x, num_points=99) if x["num_points"] else x for x in r], + ), + ("REF207", "map", lambda r: set_field(r, 0, sha256="0" * 64)), + ("REF302", "instance", lambda r: set_field(r, 0, map_element={"id": 99999, "type": "way"})), + ("REF303", "traffic_light", lambda r: set_field(r, 0, regulatory_element_id=99999)), + ( + "REF304", + "route", + lambda r: set_field( + r, 0, segments=[{"preferred_primitive_id": 99999, "primitive_ids": [99999]}] + ), + ), + ("REF305", "sensor", lambda r: _lidar(r, source_channels=["LIDAR_TOP"])), +] + + +@pytest.mark.parametrize(("rule", "table", "change"), CASES, ids=[case[0] for case in CASES]) +def test_reference_rule_fails(tmp_path: Path, rule: str, table: str, change) -> None: + sample = SampleCopy(tmp_path) + sample.edit(table, change) + report = sample.reports()[rule] + assert report.is_failed(), report + + +def test_ref306_point_labels(tmp_path: Path) -> None: + """A LiDAR `label` value that no category index has fails.""" + sample = SampleCopy(tmp_path) + sample.write_labels([[7] * 8] * 3) + report = sample.reports()["REF306"] + assert report.is_failed() and "[7]" in report.reasons[0] diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py new file mode 100644 index 00000000..6b922a7f --- /dev/null +++ b/tests/schema/v1/test_convert_v0.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import base64 +import json +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pytest +from pycocotools import mask as cocomask + +from t4_devkit.schema.v1 import SchemaName, V0Converter, load_dataset +from tests.helpers import SAMPLE_V0, SampleCopy + +set_field = SampleCopy.set_field + + +def test_load_v0_dataset() -> None: + """A 0.x dataset loads as 1.0 records, with placeholders for what 0.x never stored.""" + dataset = load_dataset(SAMPLE_V0) + tables = dataset.tables + + assert dataset.manifest is None + assert len(tables[SchemaName.SAMPLE]) == 3 + assert len(tables[SchemaName.SAMPLE_DATA]) == 9 + + # calibrated_sensor rows become sensor rows with the same tokens + sensors = {sensor.channel: sensor for sensor in tables[SchemaName.SENSOR]} + assert sorted(sensor.index for sensor in sensors.values()) == [0, 1, 2] + camera = sensors["CAM_FRONT"].camera + assert (camera.width, camera.height) == (800, 600) + assert camera.distortion_model == "plumb_bob" + lidar = sensors["LIDAR_TOP"].lidar + assert lidar.container.format == "bin" + assert lidar.container.frame_count == 3 + assert [field.name for field in lidar.fields] == ["x", "y", "z", "intensity", "channel"] + for record in tables[SchemaName.SAMPLE_DATA]: + assert dataset.get(SchemaName.SENSOR, record.sensor_token).channel in sensors + assert record.frame_index is None + + # log -> vehicle (placeholders) and map (hash from the file) + log = tables[SchemaName.LOG][0] + vehicle = dataset.get(SchemaName.VEHICLE, log.vehicle_token) + assert vehicle.name == "test_vehicle" + assert vehicle.model == V0Converter.FALLBACKS["vehicle"]["model"] + map_ = dataset.get(SchemaName.MAP, log.map_token) + assert map_.map_id == V0Converter.FALLBACKS["map"]["map_id"] + assert len(map_.sha256) == 64 + + +def test_load_v0_labels() -> None: + """Labels convert; surface_ann becomes object_ann without instance, boxed by its mask.""" + dataset = load_dataset(SAMPLE_V0) + tables = dataset.tables + + assert len(tables[SchemaName.SAMPLE_ANNOTATION]) == 4 + assert all(r.annotation_type == "manual" for r in tables[SchemaName.SAMPLE_ANNOTATION]) + assert tables[SchemaName.INSTANCE][0].name == "car_001" + # 1.0 counts 3D boxes and 2D labels (0.x tools count them differently) + for instance in tables[SchemaName.INSTANCE]: + labels = [ + r + for table in (SchemaName.SAMPLE_ANNOTATION, SchemaName.OBJECT_ANN) + for r in tables[table] + if r.instance_token == instance.token + ] + assert instance.nbr_annotations == len(labels) + + surfaces = [r for r in tables[SchemaName.OBJECT_ANN] if r.instance_token is None] + assert len(surfaces) == 2 + for record in surfaces: + category = dataset.get(SchemaName.CATEGORY, record.category_token) + assert not category.is_thing + # box on pixel edges around the decoded mask, which is (height, width) + rows, cols = np.nonzero(record.mask.decode()) + assert tuple(record.bbox) == (cols.min(), rows.min(), cols.max() + 1, rows.max() + 1) + + +def test_load_v0_category_index(tmp_path: Path) -> None: + """Categories without index get their list position; a partly set index is rejected.""" + categories = load_dataset(SAMPLE_V0).tables[SchemaName.CATEGORY] + assert [category.index for category in categories] == list(range(len(categories))) + + sample = SampleCopy(tmp_path, SAMPLE_V0) + sample.edit("category", lambda records: set_field(records, 0, index=5)) + with pytest.raises(ValueError, match="index"): + load_dataset(sample.root) + + +def test_load_v0_frames(tmp_path: Path) -> None: + """Every 0.x frame is a 1.0 sample, grouped by frame number; 0.x key samples keep their token.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + sample_data = sample.read("sample_data") + lidar2 = next(r for r in sample_data if r["filename"] == "data/LIDAR_TOP/002.pcd.bin") + lidar3 = next(r for r in sample_data if r["filename"] == "data/LIDAR_TOP/003.pcd.bin") + + def sweep(records: list[dict]) -> list[dict]: + # as the 0.x converter writes a non-key frame: linked to the next sample + for record in records: + if record["token"] == lidar2["token"]: + record.update(is_key_frame=False, sample_token=lidar3["sample_token"]) + return records + + sample.edit("sample_data", sweep) + dataset = load_dataset(sample.root) + frame = dataset.get(SchemaName.SAMPLE_DATA, lidar2["token"]) + assert frame.sample_token == lidar2["sample_token"] # its own frame's sample, not the next + assert not frame.is_key_frame + assert dataset.get(SchemaName.SAMPLE, frame.sample_token).has_key_frame # cameras are key + samples = dataset.tables[SchemaName.SAMPLE] + assert len(samples) == 3 and all(len(s.data) == 3 for s in samples) + + +def test_load_v0_per_camera_samples(tmp_path: Path) -> None: + """0.x samples of one frame (one per camera image) merge into one 1.0 sample.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + first = sample.read("sample")[0] + extra = dict(first, token="f" * 32) # sorts after `first` + sample.edit("sample", lambda records: [*records, extra]) + sample.edit( + "sample_data", + lambda records: [ + dict(r, sample_token=extra["token"]) if r["filename"] == "data/CAM_BACK/001.jpg" else r + for r in records + ], + ) + + dataset = load_dataset(sample.root) + assert len(dataset.tables[SchemaName.SAMPLE]) == 3 + channels = set(dataset.get(SchemaName.SAMPLE, first["token"]).data) + assert channels == {"LIDAR_TOP", "CAM_FRONT", "CAM_BACK"} + + +def test_load_v0_mask_only_frame_is_key(tmp_path: Path) -> None: + """A non-key 0.x frame whose only label is a semantic mask (`surface_ann`) is a key frame.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + token = sample.read("surface_ann")[0]["sample_data_token"] + sample.edit( + "sample_data", + lambda records: [ + dict(r, is_key_frame=False) if r["token"] == token else r for r in records + ], + ) + sample.edit( + "object_ann", lambda records: [r for r in records if r["sample_data_token"] != token] + ) + assert load_dataset(sample.root).get(SchemaName.SAMPLE_DATA, token).is_key_frame + + +def test_load_v0_labeled_frame_is_key(tmp_path: Path) -> None: + """A 0.x 2D label on a non-key frame makes it a key frame.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + labeled = sample.read("object_ann")[0] + sample.edit( + "sample_data", + lambda records: [ + dict(r, is_key_frame=False) if r["token"] == labeled["sample_data_token"] else r + for r in records + ], + ) + frame = load_dataset(sample.root).get(SchemaName.SAMPLE_DATA, labeled["sample_data_token"]) + assert frame.is_key_frame + + +def test_load_v0_vehicle_state() -> None: + """0.x vehicle states convert; the 0.x forward gear name becomes DRIVE.""" + states = load_dataset(SAMPLE_V0).tables[SchemaName.VEHICLE_STATE] + + assert len(states) == 3 + assert all(state.shift_state == "DRIVE" for state in states) + assert all(state.source == "vehicle_interface" for state in states) + + +def test_load_v0_capture_time(tmp_path: Path) -> None: + """An empty 0.x `data_captured` becomes the first sample time.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + sample.edit("log", lambda records: set_field(records, 0, data_captured="")) + + tables = load_dataset(sample.root).tables + first = min(s.timestamp for s in tables[SchemaName.SAMPLE]) + expected = datetime.fromtimestamp(first / 1e6, tz=timezone.utc).isoformat() + assert tables[SchemaName.LOG][0].data_captured == expected + + +@pytest.mark.parametrize( + ("table", "way_key"), + [ + ("traffic_light", "primitive_id"), + ("traffic_light_instance_map", "traffic_light_linestring_id"), + ], +) +def test_load_v0_traffic_light(tmp_path: Path, table: str, way_key: str) -> None: + """A 0.x lamp-to-way row sets the instance's lanelet2 way.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + instance_token = sample.read("instance")[0]["token"] + row = {"token": "tl0", "instance_token": instance_token, way_key: "1234"} + (sample.root / "annotation" / f"{table}.json").write_text(json.dumps([row])) + + tables = load_dataset(sample.root).tables + assert tables[SchemaName.TRAFFIC_LIGHT] == [] + map_element = tables[SchemaName.INSTANCE][0].map_element + assert (map_element.id, map_element.type) == (1234, "way") + + +def test_load_v0_source_lidar(tmp_path: Path) -> None: + """A source LiDAR takes the concatenated channel's fields and counts frames with its points.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + source = {"token": "src", "channel": "LIDAR_SOURCE", "modality": "lidar"} + calibration = {"token": "src_calib", "sensor_token": "src"} + calibration |= {"translation": [0.0, 0.0, 0.0], "rotation": [1.0, 0.0, 0.0, 0.0]} + calibration |= {"camera_intrinsic": [], "camera_distortion": []} + sample.edit("sensor", lambda records: [*records, source]) + sample.edit("calibrated_sensor", lambda records: [*records, calibration]) + + def add_info(records: list[dict]) -> list[dict]: + frames = [r for r in records if r["filename"].startswith("data/LIDAR_TOP")] + for i, record in enumerate(frames): + record["info_filename"] = f"data/LIDAR_TOP_INFO/{i}.json" + # points from the source in the first two frames only + sources = [{"sensor_token": "src", "idx_begin": 0, "length": 10 if i < 2 else 0}] + info = {"num_pts_feats": 7, "sources": sources} + (sample.root / record["info_filename"]).parent.mkdir(exist_ok=True) + (sample.root / record["info_filename"]).write_text(json.dumps(info)) + return records + + sample.edit("sample_data", add_info) + + sensors = {s.channel: s for s in load_dataset(sample.root).tables[SchemaName.SENSOR]} + concat, source_lidar = sensors["LIDAR_TOP"].lidar, sensors["LIDAR_SOURCE"].lidar + assert concat.source_channels == ["LIDAR_SOURCE"] + assert source_lidar.fields == concat.fields and len(concat.fields) == 7 + assert source_lidar.container.frame_count == 2 + + +def test_load_v0_empty_mask(tmp_path: Path) -> None: + """A box-only 0.x label with an empty placeholder mask has no mask in 1.0.""" + sample = SampleCopy(tmp_path, SAMPLE_V0) + empty = cocomask.encode(np.zeros((600, 800), dtype=np.uint8, order="F")) + # as written by some 0.x tools: (width, height) + mask = {"size": [800, 600], "counts": base64.b64encode(empty["counts"]).decode()} + sample.edit("object_ann", lambda records: set_field(records, 0, mask=mask)) + + annotation = load_dataset(sample.root).tables[SchemaName.OBJECT_ANN][0] + assert annotation.mask is None + assert list(annotation.bbox) == sample.read("object_ann")[0]["bbox"] diff --git a/tests/schema/v1/test_loader.py b/tests/schema/v1/test_loader.py new file mode 100644 index 00000000..b378da76 --- /dev/null +++ b/tests/schema/v1/test_loader.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from t4_devkit.common.serialize import serialize_dataclass, serialize_dataclasses +from t4_devkit.schema.v1 import SCHEMAS, SchemaName, Sensor, load_dataset +from tests.helpers import SAMPLE_V1, SampleCopy + + +def _copy_without_table(tmp_path: Path, name: str) -> Path: + sample = SampleCopy(tmp_path) + sample.edit( + "manifest", lambda m: dict(m, tables={k: v for k, v in m["tables"].items() if k != name}) + ) + return sample.root + + +def test_load_dataset() -> None: + """Every table loads with its 1.0 class and every field round-trips, the manifest too; + lookups by token work.""" + dataset = load_dataset(SAMPLE_V1) + + manifest = json.loads((SAMPLE_V1 / "manifest.json").read_text()) + assert serialize_dataclass(dataset.manifest) == manifest + for name in SchemaName: + records = json.loads((SAMPLE_V1 / "annotation" / name.filename).read_text()) + assert all(isinstance(r, SCHEMAS.get(name)) for r in dataset.tables[name]) + assert serialize_dataclasses(dataset.tables[name]) == records, name + + assert isinstance(dataset.get("sensor", "sensor_cam"), Sensor) + + +def test_shortcuts() -> None: + """Shortcut fields are filled from the links.""" + dataset = load_dataset(SAMPLE_V1) + + sample = dataset.tables[SchemaName.SAMPLE][0] + assert set(sample.data) == {"CAM_FRONT", "LIDAR_TOP"} + assert sample.ann_3ds + assert sample.ann_2ds + + sample_data = dataset.get("sample_data", sample.data["LIDAR_TOP"]) + assert sample_data.channel == "LIDAR_TOP" + assert dataset.get("sensor", sample_data.sensor_token).first_sd_token is not None + + # a non-key frame is a sample too, with the channels it has + non_key = next(s for s in dataset.tables[SchemaName.SAMPLE] if not s.has_key_frame) + assert set(non_key.data) == {"LIDAR_TOP"} + assert not non_key.ann_3ds and not non_key.ann_2ds + + assert dataset.tables[SchemaName.SAMPLE_ANNOTATION][0].category_name == "car" + assert {r.category_name for r in dataset.tables[SchemaName.OBJECT_ANN]} == {"car", "road"} + + +def test_missing_required_table(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match="vehicle"): + load_dataset(_copy_without_table(tmp_path, "vehicle")) + + +def test_missing_optional_table(tmp_path: Path) -> None: + dataset = load_dataset(_copy_without_table(tmp_path, "caption")) + assert dataset.tables[SchemaName.CAPTION] == [] diff --git a/tests/schema/v1/test_manifest.py b/tests/schema/v1/test_manifest.py new file mode 100644 index 00000000..d85cfe8f --- /dev/null +++ b/tests/schema/v1/test_manifest.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from attrs import evolve + +from t4_devkit.schema.v1 import ( + Capabilities, + Manifest, + SchemaName, + Task, + load_dataset, + load_manifest, +) + +SAMPLE_ROOT = Path(__file__).parents[2] / "sample" + + +def test_load_manifest_v0() -> None: + """A 0.x dataset has no manifest.""" + assert load_manifest(SAMPLE_ROOT / "t4dataset") is None + + +def test_unsupported_schema_version() -> None: + data = json.loads((SAMPLE_ROOT / "t4dataset_v1" / "manifest.json").read_text()) + data["schema_version"] = "2.0.0" + with pytest.raises(ValueError, match="Unsupported schema_version"): + Manifest(**data) + + +@pytest.mark.parametrize( + ("key", "value"), + [("is_synthetic", "yes"), ("version", -1)], +) +def test_invalid_field_type(key: str, value: object) -> None: + data = json.loads((SAMPLE_ROOT / "t4dataset_v1" / "manifest.json").read_text()) + data[key] = value + with pytest.raises((TypeError, ValueError)): + Manifest(**data) + + +def test_invalid_sha256() -> None: + data = json.loads((SAMPLE_ROOT / "t4dataset_v1" / "manifest.json").read_text()) + data["assets"][0]["sha256"] = "not-a-hash" + with pytest.raises(ValueError): + Manifest(**data) + + +def test_capabilities_prediction_and_planning() -> None: + """Prediction needs every sample to have a key frame; planning needs scenarios.""" + tables = dict(load_dataset(SAMPLE_ROOT / "t4dataset_v1").tables) + tasks = Capabilities.from_tables(tables).tasks + assert Task.PLANNING in tasks + assert Task.PREDICTION not in tasks # sample2 has no key frame + + tables[SchemaName.SAMPLE] = [evolve(s, has_key_frame=True) for s in tables[SchemaName.SAMPLE]] + tables[SchemaName.SCENARIO] = [] + tasks = Capabilities.from_tables(tables).tasks + assert Task.PREDICTION in tasks + assert Task.PLANNING not in tasks diff --git a/tests/schema/v1/test_migrate_v0.py b/tests/schema/v1/test_migrate_v0.py new file mode 100644 index 00000000..13bf8839 --- /dev/null +++ b/tests/schema/v1/test_migrate_v0.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +from t4_devkit.common.io import load_image, load_pointcloud +from t4_devkit.sanity import sanity_check +from t4_devkit.schema.v1 import SchemaName, V0Migrator +from tests.helpers import SAMPLE_V0, SampleCopy + + +def _migrate_lidar(data_root: Path, output_root: Path) -> V0Migrator: + migrator = V0Migrator(data_root, output_root) + for sensor in migrator.lidars(): + migrator.migrate_lidar(sensor) + return migrator + + +def _lidar_frames(migrator: V0Migrator) -> list: + return sorted( + (f for f in migrator.tables(SchemaName.SAMPLE_DATA) if f.fileformat.value == "parquet"), + key=lambda frame: frame.frame_index, + ) + + +def test_migrate_lidar(tmp_path: Path) -> None: + """Frames go to one Parquet file with the same values; fields that do not fit stay float32.""" + v0_frames = { + r["token"]: r["filename"] + for r in json.loads((SAMPLE_V0 / "annotation/sample_data.json").read_text()) + } + with pytest.warns(UserWarning) as warned: + migrator = _migrate_lidar(SAMPLE_V0, tmp_path / "sample") + assert [str(w.message) for w in warned] == migrator.notes + assert "'intensity' kept as float32, values in [-0.5, 115.0]" in migrator.notes[0] + + (lidar,) = migrator.lidars() + assert lidar.lidar.container.path == "data/LIDAR_TOP/LIDAR_TOP.parquet" + assert lidar.lidar.container.frame_count == 3 + # the sample's intensity and ring have fractional values + assert [(f.name, f.dtype) for f in lidar.lidar.fields][3:] == [ + ("intensity", "float32"), + ("channel", "float32"), + ] + + for i, frame in enumerate(_lidar_frames(migrator)): + assert (frame.filename, frame.frame_index, frame.num_points) == ( + lidar.lidar.container.path, + i, + 8, + ) + expected = np.fromfile(SAMPLE_V0 / v0_frames[frame.token], dtype=np.float32).reshape(-1, 5) + points = load_pointcloud(tmp_path / "sample" / frame.filename, frame_index=i) + assert np.array_equal(np.stack(list(points.values()), axis=1), expected) + + +def _semseg_copy(tmp_path: Path) -> Path: + """The 0.x sample with integer intensity, ring -1 (not recorded) and labels on frame 0.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + data_root = sample.root + frames = sorted( + (r for r in sample.read("sample_data") if "LIDAR" in r["filename"]), + key=lambda r: r["timestamp"], + ) + for record in frames: + path = data_root / record["filename"] + points = np.fromfile(path, dtype=np.float32).reshape(-1, 5) + points[:, 3] = np.arange(len(points)) + points[:, 4] = -1 + points.tofile(path) + + labels = data_root / "lidarseg/annotation/frame0.bin" + labels.parent.mkdir(parents=True) + np.array([0, 1, 2, 3, 4, 0, 1, 2], dtype=np.uint8).tofile(labels) + lidarseg = [ + { + "token": "seg0", + "filename": "lidarseg/annotation/frame0.bin", + "sample_data_token": frames[0]["token"], + } + ] + (data_root / "annotation/lidarseg.json").write_text(json.dumps(lidarseg)) + return data_root + + +def _add_unpainted(data_root: Path) -> None: + """Index the categories from 1, with `unpainted` at 0, as lidarseg datasets have them.""" + path = data_root / "annotation/category.json" + categories = [{"token": "unpainted", "name": "unpainted", "description": ""}] + categories += json.loads(path.read_text()) + for index, category in enumerate(categories): + category["index"] = index + path.write_text(json.dumps(categories)) + + +def test_migrate_lidar_labels(tmp_path: Path) -> None: + """Integer fields get 1.0 dtypes, an unrecorded ring is dropped, lidarseg gives label columns.""" + data_root = _semseg_copy(tmp_path) + with pytest.raises(ValueError, match="unpainted"): + _migrate_lidar(data_root, tmp_path / "rejected") + + _add_unpainted(data_root) + migrator = _migrate_lidar(data_root, tmp_path / "sample") + + (lidar,) = migrator.lidars() + assert [(f.name, f.dtype) for f in lidar.lidar.fields] == [ + ("x", "float32"), + ("y", "float32"), + ("z", "float32"), + ("intensity", "uint8"), + ("label", "uint8"), + ("instance", "int16"), + ] + assert "'channel' dropped" in migrator.notes[0] + path = tmp_path / "sample" / lidar.lidar.container.path + labeled = load_pointcloud(path, frame_index=0) + assert labeled["label"].tolist() == [0, 1, 2, 3, 4, 0, 1, 2] + assert labeled["instance"].tolist() == [-1] * 8 + assert "label" not in load_pointcloud(path, frame_index=1) + + +def test_migrate_camera(tmp_path: Path) -> None: + """A camera channel becomes one video; each frame is its own image, in timestamp order.""" + data_root = SampleCopy(tmp_path, SAMPLE_V0, name="v0").root + migrator = V0Migrator(data_root, tmp_path / "sample") + sensor = migrator.cameras()[0] + frames = migrator._channel_frames(sensor) + sources = [] + for i, frame in enumerate(frames): # distinct images: gray levels 50, 100, 150 + image = np.full((600, 800, 3), 50 * (i + 1), dtype=np.uint8) + Image.fromarray(image).save(data_root / frame.filename) + sources.append(image.astype(int)) + migrator.migrate_camera(sensor) + + video = sensor.camera.video + assert (video.path, video.frame_count, video.timescale) == ( + "data/CAM_FRONT/CAM_FRONT.mp4", + 3, + 1, + ) + for i, frame in enumerate(frames): + assert (frame.filename, frame.frame_index) == (video.path, i) + image = load_image(tmp_path / "sample" / video.path, frame_index=i).astype(int) + errors = [np.abs(image - source).mean() for source in sources] + assert errors[i] < 2 and np.argmin(errors) == i + + +def test_copy_images(tmp_path: Path) -> None: + """Kept images are copied with their paths; their records do not change.""" + migrator = V0Migrator(SAMPLE_V0, tmp_path / "sample") + sensor = migrator.cameras()[0] + migrator.copy_images(sensor) + + for frame in migrator._channel_frames(sensor): + assert frame.frame_index is None + assert (tmp_path / "sample" / frame.filename).read_bytes() == ( + SAMPLE_V0 / frame.filename + ).read_bytes() + + +def test_relink_half_linked_frame(tmp_path: Path) -> None: + """A frame its neighbors skip (as 0.x placeholders can be) is linked back in time order.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + first, middle, last = sorted( + (r for r in sample.read("sample_data") if r["filename"].startswith("data/CAM_FRONT/")), + key=lambda r: r["timestamp"], + ) + + def skip_middle(records: list[dict]) -> list[dict]: + for record in records: + if record["token"] == first["token"]: + record["next"] = last["token"] + if record["token"] == last["token"]: + record["prev"] = first["token"] + return records + + sample.edit("sample_data", skip_middle) + + migrator = V0Migrator(sample.root, tmp_path / "sample") + sensor = migrator.cameras()[0] + migrator.copy_images(sensor) + + frames = migrator._channel_frames(sensor) + assert [frame.next for frame in frames] == [middle["token"], last["token"], None] + assert [frame.prev for frame in frames] == [None, first["token"], middle["token"]] + assert migrator.notes == ["CAM_FRONT: next / prev relinked in time order on 2 frames"] + + +def test_migrate_lidarseg_passes_sanity(tmp_path: Path) -> None: + """A migrated lidarseg dataset passes the 1.0 rules; its instances get indices from 0.""" + data_root = _semseg_copy(tmp_path) + _add_unpainted(data_root) + output_root = tmp_path / "sample" / "1" + V0Migrator(data_root, output_root).migrate("sample", 1) + + instances = json.loads((output_root / "annotation/instance.json").read_text()) + assert [instance["index"] for instance in instances] == [0, 1, 2] + reports = sanity_check(output_root.parent.as_posix(), revision="1").reports + assert not [report.id for report in reports if report.is_failed()] + + +def test_migrate_lidar_sources(tmp_path: Path) -> None: + """`.info` sources become `source_index` (the source's `sensor.index`) and µs timestamps.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + top = next(s for s in sample.read("sensor") if s["channel"] == "LIDAR_TOP") + top_calibration = next( + c for c in sample.read("calibrated_sensor") if c["sensor_token"] == top["token"] + ) + rear_calibration = dict(top_calibration, token="rear_calibration", sensor_token="rear") + sample.edit("sensor", lambda r: [*r, dict(top, token="rear", channel="LIDAR_REAR")]) + sample.edit("calibrated_sensor", lambda r: [*r, rear_calibration]) + + data_root = sample.root + records = sample.read("sample_data") + (data_root / "data/LIDAR_TOP_INFO").mkdir() + for i, record in enumerate(r for r in records if "LIDAR" in r["filename"]): + record["info_filename"] = f"data/LIDAR_TOP_INFO/{i}.json" + info = { + "sources": [ + { + "sensor_token": top["token"], + "idx_begin": 0, + "length": 5, + "stamp": {"sec": 100, "nanosec": 1_000}, + }, + { + "sensor_token": "rear", + "idx_begin": 5, + "length": 3, + "stamp": {"sec": 100, "nanosec": 2_000_999}, + }, + ] + } + (data_root / record["info_filename"]).write_text(json.dumps(info)) + sample.edit("sample_data", lambda _: records) + + with pytest.warns(UserWarning): + migrator = _migrate_lidar(data_root, tmp_path / "sample") + sensors = {sensor.channel: sensor for sensor in migrator.tables(SchemaName.SENSOR)} + concat, rear = sensors["LIDAR_TOP"], sensors["LIDAR_REAR"] + points = load_pointcloud(tmp_path / "sample" / concat.lidar.container.path, frame_index=0) + assert points["source_index"].tolist() == [concat.index] * 5 + [rear.index] * 3 + assert points["source_timestamp"].tolist() == [100_000_001] * 5 + [100_002_000] * 3 + # the source LiDAR has no file of its own: its points are in the merged one + assert (rear.lidar.container.format.value, rear.lidar.container.path) == ("parquet", None) + assert rear.lidar.fields == concat.lidar.fields + + +def test_migrate_recalibrated_channel(tmp_path: Path) -> None: + """Two calibrations of one channel would share its file: migration stops instead.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + # LIDAR_TOP again, used by its last frame + sample.edit("calibrated_sensor", lambda r: [*r, dict(r[0], token="recalibrated")]) + lidar = [r for r in sample.read("sample_data") if "LIDAR" in r["filename"]] + last = max(lidar, key=lambda r: r["timestamp"])["token"] + sample.edit( + "sample_data", + lambda records: [ + dict(r, calibrated_sensor_token="recalibrated") if r["token"] == last else r + for r in records + ], + ) + + with pytest.warns(UserWarning), pytest.raises(FileExistsError, match="LIDAR_TOP"): + _migrate_lidar(sample.root, tmp_path / "sample") + + +def test_migrate_with_backfilled_metadata(tmp_path: Path) -> None: + """Records changed through `tables` and caller-given manifest fields are written.""" + output_root = tmp_path / "t4dataset" / "1" + migrator = V0Migrator(SAMPLE_V0, output_root) + vehicle = migrator.tables(SchemaName.VEHICLE)[0] + vehicle.model = "j6_gen2" + provenance = { + "rosbag": { + "log_file_ids": ["3145801d-4bdd-486f-b0e6-9253fca050de"], + "storage": "sqlite3", + "record_start": "2025-11-13T07:26:15Z", + "record_end": "2025-11-13T07:27:13Z", + }, + "topics": {"LIDAR_TOP": "/sensing/lidar/concatenated/pointcloud"}, + "process_config_id": None, + "process_config_version_id": None, + } + fields = { + "project_id": "x2_dev", + "provenance": provenance, + "generators": [{"name": "autoware", "version": "20250715-v4.2.1"}], + "migration_notes": ["route: dropped, lanelets not in the dataset's map"], + } + with pytest.warns(UserWarning): # the sample's fractional intensity stays float32 + migrator.migrate("t4dataset", 1, manifest_fields=fields) + + manifest = json.loads((output_root / "manifest.json").read_text()) + assert manifest["project_id"] == "x2_dev" + assert manifest["provenance"] == provenance + assert [g["name"] for g in manifest["generators"]] == ["autoware", "t4-devkit"] + # the caller's notes come after the migration's; no "first / last sample times" note + assert manifest["migration_notes"][-1].startswith("route: dropped") + assert not any("sample times" in note for note in manifest["migration_notes"]) + vehicles = json.loads((output_root / "annotation" / "vehicle.json").read_text()) + assert vehicles[0]["model"] == "j6_gen2" + reports = sanity_check(output_root.parent.as_posix(), revision="1").reports + assert not [report.id for report in reports if report.is_failed()] + + +def test_migrate_drops_v0_acceleration(tmp_path: Path) -> None: + """0.x `ego_pose.acceleration` (a raw IMU reading) is dropped, with a note.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + sample.edit( + "ego_pose", lambda records: [dict(r, acceleration=[0.0, 0.0, -9.8]) for r in records] + ) + output_root = tmp_path / "t4dataset" / "1" + with pytest.warns(UserWarning): # the sample's fractional intensity stays float32 + manifest = V0Migrator(sample.root, output_root).migrate("t4dataset", 1, keep_images=True) + + poses = json.loads((output_root / "annotation" / "ego_pose.json").read_text()) + assert all(pose["acceleration"] is None and pose["imu"] is None for pose in poses) + assert any( + note.startswith("ego_pose.acceleration dropped") for note in manifest.migration_notes + ) diff --git a/tests/schema/v1/test_registry.py b/tests/schema/v1/test_registry.py new file mode 100644 index 00000000..532886cb --- /dev/null +++ b/tests/schema/v1/test_registry.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + +from t4_devkit.schema import SCHEMAS as SCHEMAS_V0 +from t4_devkit.schema import SchemaName as SchemaNameV0 +from t4_devkit.schema.v1 import SCHEMAS, Attribute, SchemaName + + +def test_required_tables() -> None: + required = {name.value for name in SchemaName if not name.is_optional()} + assert required == {"sample", "sample_data", "ego_pose", "sensor", "scene", "log", "vehicle"} + + +def test_registries_are_separate() -> None: + """Each registry only accepts its own table names.""" + with pytest.raises(TypeError): + SCHEMAS.register(SchemaNameV0.ATTRIBUTE) + with pytest.raises(TypeError): + SCHEMAS_V0.register(SchemaName.ATTRIBUTE) + + +def test_override_table_class() -> None: + """A user can replace a table class, as documented for 0.x.""" + + @SCHEMAS.register(SchemaName.ATTRIBUTE, force=True) + class CustomAttribute(Attribute): + pass + + try: + assert SCHEMAS.get(SchemaName.ATTRIBUTE) is CustomAttribute + finally: + SCHEMAS.register(SchemaName.ATTRIBUTE, force=True)(Attribute) diff --git a/tests/schema/v1/test_tables.py b/tests/schema/v1/test_tables.py new file mode 100644 index 00000000..8555ba0a --- /dev/null +++ b/tests/schema/v1/test_tables.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import copy +import json + +import pytest + +from t4_devkit.schema.v1 import SCHEMAS, SchemaName +from tests.helpers import SAMPLE_V1 + + +def _set(path: tuple, value: object): + """Return an edit that sets `value` at `path` (keys / indices) in the record.""" + + def edit(record: dict, records: list[dict]) -> None: + target = record + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + + return edit + + +def _other_lidar_block(record: dict, records: list[dict]) -> None: + record["lidar"] = records[1]["lidar"] + + +# (table, position, edit, error): one invalid value per case; the whole table round-trips in +# test_loader.py::test_load_dataset. +CASES = { + "empty-string-link": ("sample", 0, _set(("prev",), ""), ValueError), + "frame-without-file": ("sample_data", 0, _set(("filename",), None), TypeError), + "frame-without-sample": ("sample_data", -1, _set(("sample_token",), None), TypeError), + "covariance-length": ("ego_pose", 0, _set(("covariance",), [0.0] * 35), ValueError), + "imu-length": ("ego_pose", 0, _set(("imu",), [0.0] * 3), ValueError), + "annotation-type": ("sample_annotation", 0, _set(("annotation_type",), "human"), ValueError), + "confidence-range": ("sample_annotation", 0, _set(("confidence",), 1.5), ValueError), + "instance-index-range": ("instance", 0, _set(("index",), 2**15), ValueError), + "keypoints-shape": ("keypoint", 0, _set(("keypoints",), [1.0, 2.0]), ValueError), + "scenario-end-before-start": ("scenario", 0, _set(("end_timestamp",), 0), ValueError), + "traffic-light-color": ( + "traffic_light", + 0, + _set(("elements", 0, "color"), "yellow"), + ValueError, + ), + "caption-type": ("caption", 0, _set(("caption_type",), "summary"), ValueError), + "caption-empty-text": ("caption", 0, _set(("text",), ""), ValueError), + "caption-end-before-start": ("caption", 0, _set(("end_timestamp",), 0), ValueError), + "sensor-index-range": ("sensor", 0, _set(("index",), 256), ValueError), + "sensor-index-missing": ("sensor", 0, _set(("index",), None), TypeError), + "sensor-block-modality": ("sensor", 0, _other_lidar_block, ValueError), + "video-bit-depth": ("sensor", 0, _set(("camera", "video", "bit_depth"), 12), ValueError), + "video-format": ("sensor", 0, _set(("camera", "video", "format"), "h264"), ValueError), + "camera-intrinsic": ("sensor", 0, _set(("camera", "intrinsic"), []), ValueError), + "distortion-model": ( + "sensor", + 0, + _set(("camera", "distortion_model"), "rational_polynomial"), + ValueError, + ), +} + + +@pytest.mark.parametrize(("table", "position", "edit", "error"), CASES.values(), ids=CASES.keys()) +def test_invalid_record(table: str, position: int, edit, error: type[Exception]) -> None: + """A record with one invalid value is rejected when built; unedited, it builds.""" + schema = SCHEMAS.get(SchemaName(table)) + records = json.loads((SAMPLE_V1 / "annotation" / f"{table}.json").read_text()) + record = records[position] + schema.from_dict(copy.deepcopy(record)) + edit(record, records) + with pytest.raises(error): + schema.from_dict(record) diff --git a/tests/schema/v1/test_writer.py b/tests/schema/v1/test_writer.py new file mode 100644 index 00000000..142a1efb --- /dev/null +++ b/tests/schema/v1/test_writer.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from t4_devkit.common.io import load_image, load_pointcloud +from t4_devkit.common.serialize import serialize_dataclass, serialize_dataclasses +from t4_devkit.sanity import sanity_check +from t4_devkit.schema.v1 import DatasetWriter, SchemaName, load_dataset + +SAMPLE_V1 = Path(__file__).parents[2] / "sample" / "t4dataset_v1" + + +def _manifest_info(dataset) -> dict: + """The manifest fields the caller gives: everything but the files and computed parts.""" + info = serialize_dataclass(dataset.manifest) + for name in ("tables", "assets", "capabilities"): + del info[name] + return info + + +def _sensor(dataset, channel: str): + return next(sensor for sensor in dataset.tables[SchemaName.SENSOR] if sensor.channel == channel) + + +def test_round_trip(tmp_path: Path) -> None: + """A dataset written from loaded records loads back to the same records, and passes sanity.""" + dataset = load_dataset(SAMPLE_V1) + data_root = tmp_path / "sample" # named as the sample's `dataset_id` + writer = DatasetWriter(data_root) + for directory in ("data", "map"): + writer.copy(SAMPLE_V1 / directory, directory) + manifest = writer.save(dataset.tables, _manifest_info(dataset)) + + assert manifest.capabilities == dataset.manifest.capabilities + assert any(generator.name == "t4-devkit" for generator in manifest.generators) + loaded = load_dataset(data_root) + for table in SchemaName: + assert serialize_dataclasses(loaded.tables[table]) == serialize_dataclasses( + dataset.tables[table] + ), table + + result = sanity_check(data_root.as_posix(), checksum=True) + assert not [report.id for report in result.reports if report.is_failed()] + assert {report.id for report in result.reports if not report.is_passed(strict=True)} == { + "STR001", # no version directory + "STR005", # no input_bag + "STR009", # no pointcloud map + } + + +def test_write_data(tmp_path: Path) -> None: + """LiDAR and camera frames written by the writer match the sample's sensor rows.""" + dataset = load_dataset(SAMPLE_V1) + writer = DatasetWriter(tmp_path / "sample") + + lidar = _sensor(dataset, "LIDAR_TOP") + fields = [(field.name, field.dtype) for field in lidar.lidar.fields] + frames = [ + load_pointcloud(SAMPLE_V1 / lidar.lidar.container.path, frame_index=i) + for i in range(lidar.lidar.container.frame_count) + ] + path, frame_count = writer.write_lidar("LIDAR_TOP", frames, fields) + assert (path, frame_count) == (lidar.lidar.container.path, lidar.lidar.container.frame_count) + for i, frame in enumerate(frames): + written = load_pointcloud(tmp_path / "sample" / path, frame_index=i) + assert all((written[name] == frame[name]).all() for name, _ in fields) + + camera = _sensor(dataset, "CAM_FRONT") + images = ( + load_image(SAMPLE_V1 / camera.camera.video.path, frame_index=i) + for i in range(camera.camera.video.frame_count) + ) + video = writer.write_camera("CAM_FRONT", images, frame_rate=camera.camera.video.timescale) + assert video == serialize_dataclass(camera.camera.video) + + +def test_existing_folder(tmp_path: Path) -> None: + with pytest.raises(FileExistsError): + DatasetWriter(tmp_path) + + +def test_channel_written_once(tmp_path: Path) -> None: + """A channel has one file: writing it again is refused, not overwritten.""" + writer = DatasetWriter(tmp_path / "sample") + fields = [("x", "float32")] + writer.write_lidar("LIDAR_TOP", [{"x": np.zeros(2, dtype=np.float32)}], fields) + with pytest.raises(FileExistsError, match="LIDAR_TOP"): + writer.write_lidar("LIDAR_TOP", [{"x": np.ones(2, dtype=np.float32)}], fields) diff --git a/tests/test_tier4.py b/tests/test_tier4.py index 145a60ab..0b6e9438 100644 --- a/tests/test_tier4.py +++ b/tests/test_tier4.py @@ -8,9 +8,10 @@ import pytest from t4_devkit.dataclass import Box2D, Box3D, SemanticLabel -from t4_devkit.schema import SchemaName -from t4_devkit.tier4 import DBMetadata, T4Devkit, load_metadata, load_table +from t4_devkit.schema import VisibilityLevel +from t4_devkit.tier4 import DBMetadata, T4Devkit, load_metadata from t4_devkit.typing import Quaternion +from tests.helpers import SAMPLE_V0, SampleCopy @pytest.fixture(scope="session") @@ -67,30 +68,55 @@ def test_load_metadata_with_specific_revision(self, tmp_path): assert metadata.version == "v2.0.0" -class TestLoadTable: - """Test cases for load_table function.""" - - def test_load_table_success(self, sample_dataset_path): - """Test successful table loading.""" - if not sample_dataset_path.exists(): - pytest.skip("Sample dataset not available") - - # Create a dummy annotation file - annotation_dir = sample_dataset_path.joinpath("annotation") - - result = load_table(annotation_dir.as_posix(), SchemaName.ATTRIBUTE) - - # The result should be the return value of the schema class, not its length - assert isinstance(result, list) - - def test_load_table_file_not_exists(self, tmp_path): - """Test table loading when file doesn't exist for optional schema.""" - annotation_dir = tmp_path / "annotation" - annotation_dir.mkdir() - - # Use an optional schema for this test (keypoint is actually optional) - result = load_table(str(annotation_dir), SchemaName.KEYPOINT) - assert result == [] +def test_unknown_visibility(tmp_path): + """0.x level aliases are mapped; `unavailable` is dropped, and a box with it gets + `UNAVAILABLE`.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="t4dataset") + sample.edit( + "visibility", + lambda records: SampleCopy.set_field( + SampleCopy.set_field(records, 0, level="unavailable"), # the first annotation's + 1, + level="v60-80", + ), + ) + + t4 = T4Devkit(sample.root.as_posix(), verbose=False) + assert [visibility.level for visibility in t4.visibility] == ["most", "partial", "none"] + annotation = t4.sample_annotation[0] + assert annotation.visibility_token is None + assert t4.get_box3d(annotation.token).visibility == VisibilityLevel.UNAVAILABLE + + +def test_autolabel_confidence(tmp_path): + """A 0.x autolabel becomes `auto` with the model as provenance; a box takes the confidence + of its label, and manual labels keep 1.0.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="t4dataset") + metadata = [{"name": "centerpoint-v2", "score": 0.8}] + sample.edit( + "sample_annotation", + lambda r: SampleCopy.set_field( + r, 0, automatic_annotation=True, autolabel_metadata=metadata + ), + ) + + t4 = T4Devkit(sample.root.as_posix(), verbose=False) + annotation = t4.sample_annotation[0] + assert (annotation.annotation_type, annotation.provenance) == ("auto", "centerpoint-v2") + assert t4.get_box3d(annotation.token).confidence == 0.8 + assert t4.get_box3d(t4.sample_annotation[1].token).confidence == 1.0 + + +def test_non_key_sample_boxes(): + """A frame after the last key sample gets that sample's boxes; a key frame its own.""" + t4 = T4Devkit((Path(__file__).parent / "sample/t4dataset_v1").as_posix(), verbose=False) + non_key = next( + sd for sd in t4.sample_data if not t4.get("sample", sd.sample_token).has_key_frame + ) + last_key = t4.get("sample", t4.get("sample", non_key.sample_token).prev) + assert last_key.has_key_frame + boxes = t4.get_box3ds(non_key.token) + assert len(boxes) == len(last_key.ann_3ds) > 0 class TestT4Devkit: @@ -106,25 +132,48 @@ class TestT4Devkit: - Testing performance with realistic data sizes """ - @pytest.fixture(scope="class") - def sample_t4(self): - """Create a T4Devkit instance using the real sample dataset.""" - sample_dataset_path = Path(__file__).parent / "sample/t4dataset" - if not sample_dataset_path.exists(): - pytest.skip("Sample dataset not available") - return T4Devkit(sample_dataset_path.as_posix(), verbose=False) + # Expected contents of each sample dataset: the 0.x one (converted on loading) and the 1.0 one. + EXPECTED = { + "t4dataset": { + "samples": 3, + "sample_data": 9, + "categories": 4, + "cameras": 2, + "scene_name": "minimal_test_scene", + "dataset_id": "t4dataset", + "version": None, + }, + "t4dataset_v1": { + "samples": 3, + "sample_data": 5, + "categories": 2, + "cameras": 1, + "scene_name": "sample_scene", + "dataset_id": "sample", + "version": "0", + }, + } + + @pytest.fixture(scope="class", params=sorted(EXPECTED)) + def sample_t4(self, request): + """Create a T4Devkit instance for each sample dataset.""" + sample_dataset_path = Path(__file__).parent / "sample" / request.param + t4 = T4Devkit(sample_dataset_path.as_posix(), verbose=False) + t4.expected = self.EXPECTED[request.param] + return t4 def test_dataset_structure_validation(self, sample_t4): """Validate the complete structure of the sample dataset.""" t4 = sample_t4 + expected = t4.expected - # Verify expected data counts from status.json assert len(t4.scene) == 1, "Expected exactly 1 scene" - assert len(t4.sample) == 3, "Expected exactly 3 samples" - assert len(t4.sample_data) == 9, "Expected exactly 9 sample data entries" - assert len(t4.category) == 4, "Expected exactly 4 categories" - assert len(t4.sensor) == 3, "Expected exactly 3 sensors" - assert len(t4.instance) >= 3, "Expected at least 3 instances" + assert len(t4.sample) == expected["samples"] + assert len(t4.sample_data) == expected["sample_data"] + assert len(t4.category) == expected["categories"] + assert len(t4.instance) >= 1, "Expected at least 1 instance" + assert t4.dataset_id == expected["dataset_id"] + assert t4.version == expected["version"] # Verify sensor modalities sensor_modalities = {sensor.modality for sensor in t4.sensor} @@ -136,7 +185,7 @@ def test_dataset_structure_validation(self, sample_t4): # Verify camera count camera_count = sum(1 for sensor in t4.sensor if sensor.modality == "camera") lidar_count = sum(1 for sensor in t4.sensor if sensor.modality == "lidar") - assert camera_count == 2, "Expected 2 cameras" + assert camera_count == expected["cameras"] assert lidar_count == 1, "Expected 1 LiDAR" def test_schema_reference_integrity(self, sample_t4): @@ -151,8 +200,8 @@ def test_schema_reference_integrity(self, sample_t4): for sample_data in t4.sample_data: _ = t4.get("sample", sample_data.sample_token) - # Test calibrated_sensor reference - _ = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) + # Test sensor reference + _ = t4.get("sensor", sample_data.sensor_token) # Test ego_pose reference _ = t4.get("ego_pose", sample_data.ego_pose_token) @@ -166,7 +215,8 @@ def test_schema_reference_integrity(self, sample_t4): _ = t4.get("instance", annotation.instance_token) # Test visibility reference - _ = t4.get("visibility", annotation.visibility_token) + if annotation.visibility_token is not None: + _ = t4.get("visibility", annotation.visibility_token) # Test instance -> category references for instance in t4.instance: @@ -185,8 +235,7 @@ def test_complete_sample_processing_workflow(self, sample_t4): # Process each sensor type for sample_data in sample_data_list: # Get sensor information - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) # Test sample data retrieval path, boxes, intrinsic = t4.get_sample_data(sample_data.token) @@ -196,8 +245,8 @@ def test_complete_sample_processing_workflow(self, sample_t4): # Verify sensor-specific properties if sensor.modality == "camera": assert intrinsic is not None, "Camera should have intrinsic matrix" - assert sample_data.width > 0, "Camera should have width" - assert sample_data.height > 0, "Camera should have height" + assert sensor.camera.width > 0, "Camera should have width" + assert sensor.camera.height > 0, "Camera should have height" elif sensor.modality == "lidar": assert intrinsic is None, "LiDAR should not have intrinsic matrix" @@ -243,6 +292,14 @@ def test_annotation_processing_workflows(self, sample_t4): assert hasattr(box2d, "semantic_label"), "Box should have semantic label" assert isinstance(box2d.semantic_label, SemanticLabel), "Label should be SemanticLabel" + def test_future_trajectory(self, sample_t4): + """A box gets a future path when a later annotation of its instance exists.""" + t4 = sample_t4 + + for annotation in t4.sample_annotation: + box = t4.get_box3d(annotation.token, future_seconds=10.0) + assert (box.future is not None) == (annotation.next is not None) + def test_keyframe_data_retrieval(self, sample_t4): """Test keyframe-specific data retrieval.""" t4 = sample_t4 @@ -255,8 +312,7 @@ def test_keyframe_data_retrieval(self, sample_t4): # Test 3D boxes for keyframe using sample data for sample_data in keyframe_sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) if sensor.modality == "lidar": boxes_3d = t4.get_box3ds(sample_data.token) @@ -268,8 +324,7 @@ def test_keyframe_data_retrieval(self, sample_t4): # Test 2D boxes for camera data for sample_data in keyframe_sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) if sensor.modality == "camera": boxes_2d = t4.get_box2ds(sample_data.token) @@ -312,52 +367,29 @@ def test_coordinate_system_consistency(self, sample_t4): t4 = sample_t4 for sample_data in t4.sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sample_data.calibrated_sensor_token) + sensor = t4.get("sensor", sample_data.sensor_token) ego_pose = t4.get("ego_pose", sample_data.ego_pose_token) - # Verify coordinate transformations exist - assert hasattr( - calibrated_sensor, "translation" - ), "Calibrated sensor should have translation" - assert hasattr(calibrated_sensor, "rotation"), "Calibrated sensor should have rotation" - assert hasattr(ego_pose, "translation"), "Ego pose should have translation" - assert hasattr(ego_pose, "rotation"), "Ego pose should have rotation" - # Verify coordinate dimensions - assert len(calibrated_sensor.translation) == 3, "Translation should be 3D" - assert isinstance( - calibrated_sensor.rotation, Quaternion - ), "Rotation should be quaternion" + assert len(sensor.translation) == 3, "Translation should be 3D" + assert isinstance(sensor.rotation, Quaternion), "Rotation should be quaternion" assert len(ego_pose.translation) == 3, "Translation should be 3D" assert isinstance(ego_pose.rotation, Quaternion), "Rotation should be quaternion" - def test_rendering_methods_integration(self, sample_t4): - """Test that rendering methods work with real data (without actually rendering).""" + def test_rendering(self, sample_t4, tmp_path): + """Scene, instance and pointcloud render to recording files (no viewer window).""" t4 = sample_t4 + save_dirs = {name: tmp_path / name for name in ("scene", "instance", "pointcloud")} + for save_dir in save_dirs.values(): + save_dir.mkdir() - # Test that methods exist and are callable - assert hasattr(t4, "render_scene") and callable(t4.render_scene) - assert hasattr(t4, "render_instance") and callable(t4.render_instance) - assert hasattr(t4, "render_pointcloud") and callable(t4.render_pointcloud) + t4.render_scene(future_seconds=1.0, save_dir=str(save_dirs["scene"])) + t4.render_instance(t4.instance[0].token, save_dir=str(save_dirs["instance"])) + t4.render_pointcloud(save_dir=str(save_dirs["pointcloud"])) - # We don't actually call these methods to avoid opening GUI windows - # but we can test that the data they would need is available - - # For render_scene: need sample data - assert len(t4.sample_data) > 0, "Need sample data for scene rendering" - - # For render_instance: need instances and annotations - assert len(t4.instance) > 0, "Need instances for instance rendering" - assert len(t4.sample_annotation) > 0, "Need annotations for instance rendering" - - # For render_pointcloud: need LiDAR data - lidar_data = [] - for sd in t4.sample_data: - calibrated_sensor = t4.get("calibrated_sensor", sd.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) - if sensor.modality == "lidar": - lidar_data.append(sd) - assert len(lidar_data) > 0, "Need LiDAR data for pointcloud rendering" + for save_dir in save_dirs.values(): + recordings = list(save_dir.glob("*.rrd")) + assert recordings and all(path.stat().st_size > 0 for path in recordings) def test_semantic_label_consistency(self, sample_t4): """Test semantic label consistency across annotations.""" @@ -419,7 +451,7 @@ def test_data_file_paths(self, sample_t4): assert path.startswith(t4.data_root), f"Path should start with data root: {path}" # Should end with expected file extension - expected_extensions = [".jpg", ".png", ".pcd.bin", ".bin"] + expected_extensions = [".jpg", ".png", ".pcd.bin", ".bin", ".mp4", ".parquet"] assert any( path.endswith(ext) for ext in expected_extensions ), f"Unexpected file extension: {path}" @@ -434,7 +466,7 @@ def test_complete_integration_workflow(self, sample_t4): # Start with scene scene = t4.scene[0] - assert scene.name == "minimal_test_scene" + assert scene.name == t4.expected["scene_name"] # Get first sample first_sample = t4.get("sample", scene.first_sample_token) @@ -447,8 +479,7 @@ def test_complete_integration_workflow(self, sample_t4): # Process camera data camera_data = None for sd in sample_data_list: - calibrated_sensor = t4.get("calibrated_sensor", sd.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sd.sensor_token) if sensor.modality == "camera": camera_data = sd break @@ -456,7 +487,7 @@ def test_complete_integration_workflow(self, sample_t4): if camera_data: # Get camera sample data path, boxes, intrinsic = t4.get_sample_data(camera_data.token) - assert path.endswith(".jpg") + assert path.endswith((".jpg", ".mp4")) assert intrinsic is not None # Get 2D boxes for this camera @@ -466,8 +497,7 @@ def test_complete_integration_workflow(self, sample_t4): # Process LiDAR data lidar_data = None for sd in sample_data_list: - calibrated_sensor = t4.get("calibrated_sensor", sd.calibrated_sensor_token) - sensor = t4.get("sensor", calibrated_sensor.sensor_token) + sensor = t4.get("sensor", sd.sensor_token) if sensor.modality == "lidar": lidar_data = sd break @@ -475,7 +505,7 @@ def test_complete_integration_workflow(self, sample_t4): if lidar_data: # Get LiDAR sample data path, boxes, intrinsic = t4.get_sample_data(lidar_data.token) - assert path.endswith((".pcd.bin", ".bin")) + assert path.endswith((".pcd.bin", ".bin", ".parquet")) assert intrinsic is None # Get 3D boxes for this sample using LiDAR data diff --git a/tests/viewer/test_viewer.py b/tests/viewer/test_viewer.py index d69e25e2..d3a8bc44 100644 --- a/tests/viewer/test_viewer.py +++ b/tests/viewer/test_viewer.py @@ -1,14 +1,21 @@ from __future__ import annotations +from pathlib import Path + import numpy as np import pytest from pyquaternion import Quaternion from t4_devkit.dataclass import LidarPointCloud -from t4_devkit.schema import CalibratedSensor, EgoPose, Sensor +from t4_devkit.schema import EgoPose +from t4_devkit.schema.v1 import Sensor from t4_devkit.viewer import EntityPath, format_entity from t4_devkit.viewer import viewer as viewer_module +SAMPLE_V1_SENSOR = ( + Path(__file__).parents[1] / "sample" / "t4dataset_v1" / "annotation" / "sensor.json" +) + def test_format_entity() -> None: """Test `format_entity(...)` function.""" @@ -153,17 +160,10 @@ def test_render_calibration(dummy_viewer, dummy_camera_calibration) -> None: camera_intrinsic=camera_intrinsic, ) - # with `Sensor` and `CalibratedSensor` - sensor = Sensor(token="sensor", channel="camera", modality="camera") - calibration = CalibratedSensor( - token="sensor_calibration", - sensor_token="sensor", - translation=translation, - rotation=rotation, - camera_intrinsic=camera_intrinsic, - camera_distortion=[0, 0, 0, 0, 0], - ) - dummy_viewer.render_calibration(sensor, calibration) + # with a `Sensor` record (camera of the 1.0 sample) + sensor = Sensor.from_json(SAMPLE_V1_SENSOR)[0] + assert sensor.camera is not None + dummy_viewer.render_calibration(sensor) def test_render_map(dummy_viewer, dummy_lanelet_path) -> None: