diff --git a/docs/cli/t4sanity.md b/docs/cli/t4sanity.md index ee5209bf..cbecaf16 100644 --- a/docs/cli/t4sanity.md +++ b/docs/cli/t4sanity.md @@ -1,4 +1,4 @@ -`t4sanity` performs sanity checks on T4 datasets, reporting any issues regarding the [dataset requirements](../schema/requirement.md). +`t4sanity` performs sanity checks on T4 datasets, reporting any issues regarding the dataset requirements: [0.x](../schema/requirement.md) or [1.0](../schema/v1/requirement.md), chosen by the dataset (1.0 if it has a `manifest.json`). ```shell $ t4sanity -h @@ -15,6 +15,7 @@ $ t4sanity -h │ --exclude -e TEXT Exclude specific rules or rule groups. [default: None] │ │ --strict -s Indicates whether warnings are treated as failures. │ │ --fix -f Attempt to fix the issues reported by the sanity check. │ +│ --checksum -c Verify the SHA-256 of every file listed in manifest.json (1.0 datasets). │ │ --install-completion Install completion for the current shell. │ │ --show-completion Show completion for the current shell, to copy it or customize the installation. │ │ --help -h Show this message and exit. │ diff --git a/t4_devkit/cli/sanity.py b/t4_devkit/cli/sanity.py index 3a06758d..3efc29c1 100644 --- a/t4_devkit/cli/sanity.py +++ b/t4_devkit/cli/sanity.py @@ -45,8 +45,17 @@ def main( fix: bool = typer.Option( False, "-f", "--fix", help="Attempt to fix the issues reported by the sanity check." ), + checksum: bool = typer.Option( + False, + "-c", + "--checksum", + help="Verify the SHA-256 of every file listed in manifest.json (1.0 datasets). " + "Reads every file, including videos and rosbags.", + ), ) -> None: - result = sanity_check(data_root=data_root, revision=revision, excludes=excludes, fix=fix) + result = sanity_check( + data_root=data_root, revision=revision, excludes=excludes, fix=fix, checksum=checksum + ) print_sanity_result(result, strict=strict) diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index e4a88d77..53af1e98 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -6,17 +6,33 @@ from returns.maybe import Maybe, Nothing, Some -from .result import make_report, make_skipped +from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS + +from .result import Reason, make_report, make_skipped if TYPE_CHECKING: + from t4_devkit.schema.v1 import SchemaName as V1SchemaName + from .context import SanityContext - from .result import Reason, Report + from .result import Report RuleID = NewType("RuleID", str) RuleName = NewType("RuleName", str) +def record_name(table: V1SchemaName) -> str: + """Return the class name of a 1.0 table's records (e.g. "SampleAnnotation"), for reasons.""" + return V1_SCHEMAS.get(table).__name__ + + +class SchemaVersion(str, Enum): + """Dataset schema versions a rule can apply to.""" + + V0 = "0.x" + V1 = "1.0" + + class Severity(str, Enum): """Severity levels for sanity checkers.""" @@ -33,12 +49,17 @@ def is_error(self) -> bool: class Checker(ABC): - """Base class for sanity checkers.""" + """Base class for sanity checkers. + + Attributes: + versions (tuple[SchemaVersion, ...]): Schema versions the rule applies to; 0.x by default. + """ id: RuleID name: RuleName description: str severity: Severity + versions: tuple[SchemaVersion, ...] = (SchemaVersion.V0,) def __call__(self, context: SanityContext, fix: bool = False) -> Report: """Run the checker and return a report. @@ -94,3 +115,36 @@ def fix(self, context: SanityContext) -> bool: True if the issue was fixed, False otherwise. """ return False + + +class ManifestChecker(Checker): + """Base class for 1.0 rules that read `manifest.json`: skipped when it cannot be read or is + not valid (reported by STR010 and FMT020).""" + + versions = (SchemaVersion.V1,) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match context.manifest: + case Some(_): + return Nothing + return Maybe.from_value(Reason("No valid 'manifest.json' (see STR010, FMT020)")) + + +class RecordsChecker(ManifestChecker): + """Base class for 1.0 rules on records: skipped unless every table they read is listed (or + optional) and builds (reported by STR014 and the format rules). + + Attributes: + tables (tuple[V1SchemaName, ...]): The tables the rule reads. + """ + + tables: tuple[V1SchemaName, ...] + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + skip = super().can_skip(context) + if skip != Nothing: + return skip + invalid = [table.value for table in self.tables if context.load_records(table) == Nothing] + if invalid: + return Maybe.from_value(Reason(f"Tables not listed or not valid: {', '.join(invalid)}")) + return Nothing diff --git a/t4_devkit/sanity/context.py b/t4_devkit/sanity/context.py index 28537683..485195a1 100644 --- a/t4_devkit/sanity/context.py +++ b/t4_devkit/sanity/context.py @@ -1,30 +1,50 @@ from __future__ import annotations +import hashlib +from functools import cached_property from pathlib import Path -from attrs import define -from returns.maybe import Maybe, Some +from attrs import define, field +from returns.converters import result_to_maybe +from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful +from returns.result import safe from typing_extensions import TYPE_CHECKING, Self from t4_devkit import DBMetadata +from t4_devkit.lanelet import LaneletParser from t4_devkit.schema import SchemaName +from t4_devkit.schema.v1 import MANIFEST_FILENAME, SCHEMAS, Manifest +from t4_devkit.schema.v1 import SchemaName as V1SchemaName -from .safety import load_metadata_safe, save_json_safe +from .checker import SchemaVersion +from .safety import load_json_safe, load_manifest_safe, load_metadata_safe, save_json_safe if TYPE_CHECKING: from t4_devkit.typing import PathLike -@define +@define(slots=False) class SanityContext: + """What the checkers read: the dataset paths, and for 1.0 its manifest and tables, read once. + + Attributes: + metadata (Maybe[DBMetadata]): Dataset root, ID and version directory. + checksum (bool): Whether to verify the SHA-256 of the files listed in `manifest.json`. + """ + metadata: Maybe[DBMetadata] + checksum: bool = field(default=False, kw_only=True) + _tables: dict[V1SchemaName, Maybe[list[dict]]] = field(factory=dict, init=False, repr=False) + _records: dict[V1SchemaName, Maybe[list]] = field(factory=dict, init=False, repr=False) @classmethod - def from_path(cls, data_root: PathLike, revision: str | None = None) -> Self: + def from_path( + cls, data_root: PathLike, revision: str | None = None, *, checksum: bool = False + ) -> Self: metadata_result = load_metadata_safe(data_root, revision=revision) metadata = metadata_result.unwrap() if is_successful(metadata_result) else None - return cls(Maybe.from_optional(metadata)) + return cls(Maybe.from_optional(metadata), checksum=checksum) @property def data_root(self) -> Maybe[Path]: @@ -66,12 +86,98 @@ def status_json(self) -> Maybe[Path]: """Return the path to status JSON file, which is 'status.json'.""" return self.metadata.map(lambda m: Path(m.data_root).joinpath("status.json")) + @property + def manifest_json(self) -> Maybe[Path]: + """Return the path to `manifest.json`.""" + return self.metadata.map(lambda m: Path(m.data_root).joinpath(MANIFEST_FILENAME)) + + @property + def schema_version(self) -> SchemaVersion: + """Return 1.0 if `manifest.json` exists at the dataset root, 0.x otherwise.""" + match self.manifest_json: + case Some(x) if x.exists(): + return SchemaVersion.V1 + return SchemaVersion.V0 + + @cached_property + def raw_manifest(self) -> Maybe[dict]: + """Return `manifest.json` as read, or Nothing if it cannot be read.""" + match self.manifest_json: + case Some(x) if x.exists(): + result = load_json_safe(x) + return Some(result.unwrap()) if is_successful(result) else Nothing + return Nothing + + @cached_property + def manifest(self) -> Maybe[Manifest]: + """Return the manifest, or Nothing if it cannot be read or is not valid.""" + match self.raw_manifest: + case Some(data): + return result_to_maybe(load_manifest_safe(data)) + return Nothing + + def to_table_file(self, table: V1SchemaName) -> Maybe[Path]: + """Return the path of a 1.0 table file listed in `manifest.json`, or Nothing if unlisted.""" + match self.manifest, self.data_root: + case Some(manifest), Some(data_root) if table.value in manifest.tables: + return Some(data_root.joinpath(manifest.tables[table.value].path)) + return Nothing + + def load_table(self, table: V1SchemaName) -> Maybe[list[dict]]: + """Return the raw records of a 1.0 table as read (read once), or Nothing if unlisted or + unreadable.""" + if table not in self._tables: + match self.to_table_file(table): + case Some(path): + self._tables[table] = result_to_maybe(load_json_safe(path)) + case _: + self._tables[table] = Nothing + return self._tables[table] + + def load_records(self, table: V1SchemaName) -> Maybe[list]: + """Return the 1.0 records of a table (built once): empty for an unlisted optional table, + Nothing if the table is required and unlisted, unreadable, or has an invalid record + (reported by STR014 and the format rules).""" + if table not in self._records: + self._records[table] = self._build_records(table) + return self._records[table] + + @cached_property + def lanelet_map(self) -> Maybe[LaneletParser]: + """Return the lanelet2 map of the dataset (read once): the file of the first `Map` + record, or Nothing if there is none or it cannot be read.""" + match self.load_records(V1SchemaName.MAP), self.data_root: + case Some([map_record, *_]), Some(data_root): + return result_to_maybe(_parse_lanelet_safe(data_root.joinpath(map_record.filename))) + return Nothing + + def save_table(self, table: V1SchemaName, records: list[dict]) -> bool: + """Save the raw records of a 1.0 table, and its new SHA-256 in `manifest.json`. + + Returns: + True if both files were written. + """ + filepath = self.to_table_file(table).value_or(None) + if filepath is None or not is_successful(save_json_safe(records, filepath)): + return False + manifest = self.raw_manifest.unwrap() + manifest["tables"][table.value]["sha256"] = hashlib.sha256( + filepath.read_bytes() + ).hexdigest() + saved = is_successful(save_json_safe(manifest, self.manifest_json.unwrap())) + # read again on next use + self._tables.pop(table, None) + self._records.pop(table, None) + return saved + def to_schema_file(self, schema: SchemaName) -> Maybe[Path]: """Convert schema name to file path, which is /annotation/.json.""" return self.annotation_dir.map(lambda ann: ann.joinpath(schema.filename)) def save_records(self, schema: SchemaName, records: list[dict]) -> bool: - """Save schema data to file.""" + """Save schema data to file (for 1.0, also its SHA-256 in `manifest.json`).""" + if self.schema_version == SchemaVersion.V1: + return self.save_table(V1SchemaName(schema.value), records) match self.to_schema_file(schema): case Some(filepath): result = save_json_safe(records, filepath) @@ -81,3 +187,24 @@ def save_records(self, schema: SchemaName, records: list[dict]) -> bool: print(f"Error saving schema: {result.failure()}") return False return False + + def _build_records(self, table: V1SchemaName) -> Maybe[list]: + match self.manifest: + case Some(manifest): + if table.value not in manifest.tables: + return Some([]) if table.is_optional() else Nothing + match self.load_table(table): + case Some(records): + return result_to_maybe(_build_safe(table, records)) + return Nothing + + +@safe +def _build_safe(table: V1SchemaName, records: list[dict]) -> list: + """Build the 1.0 records of a table; fails on the first invalid record.""" + return [SCHEMAS.get(table).from_dict(record) for record in records] + + +@safe +def _parse_lanelet_safe(path: Path) -> LaneletParser: + return LaneletParser(path.as_posix(), verbose=False) diff --git a/t4_devkit/sanity/format/__init__.py b/t4_devkit/sanity/format/__init__.py index d7af72ab..059af070 100644 --- a/t4_devkit/sanity/format/__init__.py +++ b/t4_devkit/sanity/format/__init__.py @@ -19,3 +19,24 @@ from .fmt017 import * # noqa from .fmt018 import * # noqa from .fmt019 import * # noqa +from .fmt020 import * # noqa +from .fmt021 import * # noqa +from .fmt022 import * # noqa +from .fmt023 import * # noqa +from .fmt024 import * # noqa +from .fmt025 import * # noqa +from .fmt026 import * # noqa +from .fmt027 import * # noqa +from .fmt028 import * # noqa +from .fmt029 import * # noqa +from .fmt030 import * # noqa +from .fmt031 import * # noqa +from .fmt032 import * # noqa +from .fmt033 import * # noqa +from .fmt034 import * # noqa +from .fmt035 import * # noqa +from .fmt036 import * # noqa +from .fmt037 import * # noqa +from .fmt038 import * # noqa +from .fmt039 import * # noqa +from .fmt040 import * # noqa diff --git a/t4_devkit/sanity/format/base.py b/t4_devkit/sanity/format/base.py index a776cff5..ae6977cc 100644 --- a/t4_devkit/sanity/format/base.py +++ b/t4_devkit/sanity/format/base.py @@ -6,8 +6,10 @@ from returns.pipeline import is_successful from t4_devkit.schema import SCHEMAS, SchemaName +from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS +from t4_devkit.schema.v1 import SchemaName as V1SchemaName -from ..checker import Checker +from ..checker import Checker, ManifestChecker from ..result import Reason from ..safety import load_json_safe, load_schema_safe @@ -55,3 +57,43 @@ def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason] | No if not is_successful(conversion): failures.append(Reason(f"[{schema.name}] {record['token']}: {conversion.failure()}")) return failures if failures else None + + +class TableFieldChecker(ManifestChecker): + """Base class for 1.0 format checkers: every record of a table builds into its 1.0 class. + + Attributes: + table (V1SchemaName): The 1.0 table to check. + """ + + table: V1SchemaName + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + skip = super().can_skip(context) + if skip != Nothing: + return skip + if ( + not self.table.is_optional() + and self.table.value not in context.manifest.unwrap().tables + ): + return Maybe.from_value(Reason(f"'{self.table.value}' not listed (see STR014)")) + return Nothing + + def check(self, context: SanityContext) -> list[Reason] | None: + if self.table.value not in context.manifest.unwrap().tables: + return None # optional table not present + match context.load_table(self.table): + case Some(records): + return _build_v1_records(self.table, records) + return [Reason(f"'{context.to_table_file(self.table).unwrap()}' cannot be read")] + + +def _build_v1_records(table: V1SchemaName, records: list[dict]) -> list[Reason] | None: + module = V1_SCHEMAS.get(table) + failures = [] + for record in records: + conversion = load_schema_safe(module, record) + if not is_successful(conversion): + token = record.get("token") if isinstance(record, dict) else None + failures.append(Reason(f"[{table.name}] {token}: {conversion.failure()}")) + return failures if failures else None diff --git a/t4_devkit/sanity/format/fmt020.py b/t4_devkit/sanity/format/fmt020.py new file mode 100644 index 00000000..059b0dc6 --- /dev/null +++ b/t4_devkit/sanity/format/fmt020.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful + +from t4_devkit.schema.v1 import MANIFEST_FILENAME + +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity +from ..registry import CHECKERS +from ..result import Reason +from ..safety import load_manifest_safe + +if TYPE_CHECKING: + from ..context import SanityContext + +__all__ = ["FMT020"] + + +@CHECKERS.register() +class FMT020(Checker): + """A checker of FMT020.""" + + id = RuleID("FMT020") + name = RuleName("manifest-field") + severity = Severity.ERROR + description = "All 'manifest.json' fields are valid." + versions = (SchemaVersion.V1,) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + match context.raw_manifest: + case Some(_): + return Nothing + return Maybe.from_value(Reason(f"'{MANIFEST_FILENAME}' cannot be read (see STR010)")) + + def check(self, context: SanityContext) -> list[Reason] | None: + result = load_manifest_safe(context.raw_manifest.unwrap()) + return None if is_successful(result) else [Reason(f"{result.failure()}")] diff --git a/t4_devkit/sanity/format/fmt021.py b/t4_devkit/sanity/format/fmt021.py new file mode 100644 index 00000000..d918b693 --- /dev/null +++ b/t4_devkit/sanity/format/fmt021.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT021"] + + +@CHECKERS.register() +class FMT021(TableFieldChecker): + """A checker of FMT021.""" + + id = RuleID("FMT021") + name = RuleName("sample-field") + severity = Severity.ERROR + description = "All 'Sample' records are valid." + table = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/format/fmt022.py b/t4_devkit/sanity/format/fmt022.py new file mode 100644 index 00000000..09cf313f --- /dev/null +++ b/t4_devkit/sanity/format/fmt022.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT022"] + + +@CHECKERS.register() +class FMT022(TableFieldChecker): + """A checker of FMT022.""" + + id = RuleID("FMT022") + name = RuleName("sample-data-field") + severity = Severity.ERROR + description = "All 'SampleData' records are valid." + table = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/format/fmt023.py b/t4_devkit/sanity/format/fmt023.py new file mode 100644 index 00000000..01ba7b4a --- /dev/null +++ b/t4_devkit/sanity/format/fmt023.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT023"] + + +@CHECKERS.register() +class FMT023(TableFieldChecker): + """A checker of FMT023.""" + + id = RuleID("FMT023") + name = RuleName("ego-pose-field") + severity = Severity.ERROR + description = "All 'EgoPose' records are valid." + table = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/format/fmt024.py b/t4_devkit/sanity/format/fmt024.py new file mode 100644 index 00000000..b8e8d774 --- /dev/null +++ b/t4_devkit/sanity/format/fmt024.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT024"] + + +@CHECKERS.register() +class FMT024(TableFieldChecker): + """A checker of FMT024.""" + + id = RuleID("FMT024") + name = RuleName("sensor-field") + severity = Severity.ERROR + description = ( + "All 'Sensor' records are valid, including exactly one modality block, matching 'modality'." + ) + table = SchemaName.SENSOR diff --git a/t4_devkit/sanity/format/fmt025.py b/t4_devkit/sanity/format/fmt025.py new file mode 100644 index 00000000..2355a9e6 --- /dev/null +++ b/t4_devkit/sanity/format/fmt025.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT025"] + + +@CHECKERS.register() +class FMT025(TableFieldChecker): + """A checker of FMT025.""" + + id = RuleID("FMT025") + name = RuleName("scene-field") + severity = Severity.ERROR + description = "All 'Scene' records are valid." + table = SchemaName.SCENE diff --git a/t4_devkit/sanity/format/fmt026.py b/t4_devkit/sanity/format/fmt026.py new file mode 100644 index 00000000..990883a0 --- /dev/null +++ b/t4_devkit/sanity/format/fmt026.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT026"] + + +@CHECKERS.register() +class FMT026(TableFieldChecker): + """A checker of FMT026.""" + + id = RuleID("FMT026") + name = RuleName("log-field") + severity = Severity.ERROR + description = "All 'Log' records are valid." + table = SchemaName.LOG diff --git a/t4_devkit/sanity/format/fmt027.py b/t4_devkit/sanity/format/fmt027.py new file mode 100644 index 00000000..99fd9ac6 --- /dev/null +++ b/t4_devkit/sanity/format/fmt027.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT027"] + + +@CHECKERS.register() +class FMT027(TableFieldChecker): + """A checker of FMT027.""" + + id = RuleID("FMT027") + name = RuleName("vehicle-field") + severity = Severity.ERROR + description = "All 'Vehicle' records are valid." + table = SchemaName.VEHICLE diff --git a/t4_devkit/sanity/format/fmt028.py b/t4_devkit/sanity/format/fmt028.py new file mode 100644 index 00000000..313becec --- /dev/null +++ b/t4_devkit/sanity/format/fmt028.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT028"] + + +@CHECKERS.register() +class FMT028(TableFieldChecker): + """A checker of FMT028.""" + + id = RuleID("FMT028") + name = RuleName("map-field") + severity = Severity.ERROR + description = "All 'Map' records are valid." + table = SchemaName.MAP diff --git a/t4_devkit/sanity/format/fmt029.py b/t4_devkit/sanity/format/fmt029.py new file mode 100644 index 00000000..9d2b9735 --- /dev/null +++ b/t4_devkit/sanity/format/fmt029.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT029"] + + +@CHECKERS.register() +class FMT029(TableFieldChecker): + """A checker of FMT029.""" + + id = RuleID("FMT029") + name = RuleName("vehicle-state-field") + severity = Severity.ERROR + description = "All 'VehicleState' records are valid." + table = SchemaName.VEHICLE_STATE diff --git a/t4_devkit/sanity/format/fmt030.py b/t4_devkit/sanity/format/fmt030.py new file mode 100644 index 00000000..8916fa0d --- /dev/null +++ b/t4_devkit/sanity/format/fmt030.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT030"] + + +@CHECKERS.register() +class FMT030(TableFieldChecker): + """A checker of FMT030.""" + + id = RuleID("FMT030") + name = RuleName("category-field") + severity = Severity.ERROR + description = "All 'Category' records are valid." + table = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/format/fmt031.py b/t4_devkit/sanity/format/fmt031.py new file mode 100644 index 00000000..4d962100 --- /dev/null +++ b/t4_devkit/sanity/format/fmt031.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT031"] + + +@CHECKERS.register() +class FMT031(TableFieldChecker): + """A checker of FMT031.""" + + id = RuleID("FMT031") + name = RuleName("attribute-field") + severity = Severity.ERROR + description = "All 'Attribute' records are valid." + table = SchemaName.ATTRIBUTE diff --git a/t4_devkit/sanity/format/fmt032.py b/t4_devkit/sanity/format/fmt032.py new file mode 100644 index 00000000..67f8af2f --- /dev/null +++ b/t4_devkit/sanity/format/fmt032.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT032"] + + +@CHECKERS.register() +class FMT032(TableFieldChecker): + """A checker of FMT032.""" + + id = RuleID("FMT032") + name = RuleName("visibility-field") + severity = Severity.ERROR + description = "All 'Visibility' records are valid." + table = SchemaName.VISIBILITY diff --git a/t4_devkit/sanity/format/fmt033.py b/t4_devkit/sanity/format/fmt033.py new file mode 100644 index 00000000..1666c7e7 --- /dev/null +++ b/t4_devkit/sanity/format/fmt033.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT033"] + + +@CHECKERS.register() +class FMT033(TableFieldChecker): + """A checker of FMT033.""" + + id = RuleID("FMT033") + name = RuleName("instance-field") + severity = Severity.ERROR + description = "All 'Instance' records are valid." + table = SchemaName.INSTANCE diff --git a/t4_devkit/sanity/format/fmt034.py b/t4_devkit/sanity/format/fmt034.py new file mode 100644 index 00000000..5aca2a34 --- /dev/null +++ b/t4_devkit/sanity/format/fmt034.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT034"] + + +@CHECKERS.register() +class FMT034(TableFieldChecker): + """A checker of FMT034.""" + + id = RuleID("FMT034") + name = RuleName("sample-annotation-field") + severity = Severity.ERROR + description = "All 'SampleAnnotation' records are valid." + table = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/format/fmt035.py b/t4_devkit/sanity/format/fmt035.py new file mode 100644 index 00000000..28e323f7 --- /dev/null +++ b/t4_devkit/sanity/format/fmt035.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT035"] + + +@CHECKERS.register() +class FMT035(TableFieldChecker): + """A checker of FMT035.""" + + id = RuleID("FMT035") + name = RuleName("object-ann-field") + severity = Severity.ERROR + description = "All 'ObjectAnn' records are valid." + table = SchemaName.OBJECT_ANN diff --git a/t4_devkit/sanity/format/fmt036.py b/t4_devkit/sanity/format/fmt036.py new file mode 100644 index 00000000..1cf64d8e --- /dev/null +++ b/t4_devkit/sanity/format/fmt036.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT036"] + + +@CHECKERS.register() +class FMT036(TableFieldChecker): + """A checker of FMT036.""" + + id = RuleID("FMT036") + name = RuleName("keypoint-field") + severity = Severity.ERROR + description = "All 'Keypoint' records are valid." + table = SchemaName.KEYPOINT diff --git a/t4_devkit/sanity/format/fmt037.py b/t4_devkit/sanity/format/fmt037.py new file mode 100644 index 00000000..1bf823b0 --- /dev/null +++ b/t4_devkit/sanity/format/fmt037.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT037"] + + +@CHECKERS.register() +class FMT037(TableFieldChecker): + """A checker of FMT037.""" + + id = RuleID("FMT037") + name = RuleName("traffic-light-field") + severity = Severity.ERROR + description = "All 'TrafficLight' records are valid." + table = SchemaName.TRAFFIC_LIGHT diff --git a/t4_devkit/sanity/format/fmt038.py b/t4_devkit/sanity/format/fmt038.py new file mode 100644 index 00000000..57faa26a --- /dev/null +++ b/t4_devkit/sanity/format/fmt038.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT038"] + + +@CHECKERS.register() +class FMT038(TableFieldChecker): + """A checker of FMT038.""" + + id = RuleID("FMT038") + name = RuleName("route-field") + severity = Severity.ERROR + description = "All 'Route' records are valid." + table = SchemaName.ROUTE diff --git a/t4_devkit/sanity/format/fmt039.py b/t4_devkit/sanity/format/fmt039.py new file mode 100644 index 00000000..a596e909 --- /dev/null +++ b/t4_devkit/sanity/format/fmt039.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT039"] + + +@CHECKERS.register() +class FMT039(TableFieldChecker): + """A checker of FMT039.""" + + id = RuleID("FMT039") + name = RuleName("scenario-field") + severity = Severity.ERROR + description = "All 'Scenario' records are valid." + table = SchemaName.SCENARIO diff --git a/t4_devkit/sanity/format/fmt040.py b/t4_devkit/sanity/format/fmt040.py new file mode 100644 index 00000000..348443ad --- /dev/null +++ b/t4_devkit/sanity/format/fmt040.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RuleID, RuleName, Severity +from ..registry import CHECKERS +from .base import TableFieldChecker + +__all__ = ["FMT040"] + + +@CHECKERS.register() +class FMT040(TableFieldChecker): + """A checker of FMT040.""" + + id = RuleID("FMT040") + name = RuleName("caption-field") + severity = Severity.ERROR + description = "All 'Caption' records are valid." + table = SchemaName.CAPTION diff --git a/t4_devkit/sanity/record/__init__.py b/t4_devkit/sanity/record/__init__.py index 92398fa0..3af0f9fd 100644 --- a/t4_devkit/sanity/record/__init__.py +++ b/t4_devkit/sanity/record/__init__.py @@ -7,3 +7,25 @@ from .rec005 import * # noqa from .rec006 import * # noqa from .rec007 import * # noqa +from .rec008 import * # noqa +from .rec009 import * # noqa +from .rec010 import * # noqa +from .rec011 import * # noqa +from .rec012 import * # noqa +from .rec013 import * # noqa +from .rec014 import * # noqa +from .rec015 import * # noqa +from .rec016 import * # noqa +from .rec017 import * # noqa +from .rec018 import * # noqa +from .rec019 import * # noqa +from .rec020 import * # noqa +from .rec021 import * # noqa +from .rec022 import * # noqa +from .rec023 import * # noqa +from .rec024 import * # noqa +from .rec025 import * # noqa +from .rec026 import * # noqa +from .rec027 import * # noqa +from .rec028 import * # noqa +from .rec029 import * # noqa diff --git a/t4_devkit/sanity/record/base.py b/t4_devkit/sanity/record/base.py index 57984ea9..e712957f 100644 --- a/t4_devkit/sanity/record/base.py +++ b/t4_devkit/sanity/record/base.py @@ -4,14 +4,19 @@ from typing import TYPE_CHECKING from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful -from t4_devkit.schema import SchemaName +from t4_devkit.schema import SchemaName, SensorModality +from t4_devkit.schema.v1 import FileFormat, LidarContainerFormat, Task +from t4_devkit.schema.v1 import SchemaName as V1SchemaName from ..checker import Checker from ..result import Reason from ..safety import load_json_safe if TYPE_CHECKING: + from t4_devkit.schema.v1 import Sensor + from ..context import SanityContext @@ -33,6 +38,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case Some(x): if not x.exists(): return Maybe.from_value(Reason(f"Missing {self.schema.filename}")) + elif not is_successful(load_json_safe(x)): + return Maybe.from_value(Reason(f"Cannot read {self.schema.filename}")) else: return Nothing case _: @@ -54,3 +61,105 @@ def check_count(self, records: list[dict]) -> list[Reason] | None: A list of reasons for any issues found, otherwise None. """ pass + + +def channel_file(sensor: Sensor) -> tuple[str, int] | None: + """Return the file and frame count of a channel stored in one file, or None for one file per + frame.""" + if sensor.camera is not None and sensor.camera.video is not None: + return sensor.camera.video.path, sensor.camera.video.frame_count + if sensor.lidar is not None and sensor.lidar.container.path is not None: + return sensor.lidar.container.path, sensor.lidar.container.frame_count + return None + + +# `sample_data.fileformat` values for each LiDAR container format. +LIDAR_FRAME_FORMATS = { + LidarContainerFormat.PARQUET: {FileFormat.PARQUET.value}, + LidarContainerFormat.PCD: {FileFormat.PCD.value}, + LidarContainerFormat.BIN: {FileFormat.BIN.value, FileFormat.PCDBIN.value}, +} + + +def frame_formats(sensor: Sensor) -> set[str]: + """Return the `sample_data.fileformat` values a sensor's storage allows (any for radar, + undefined in 1.0).""" + if sensor.camera is not None: + if sensor.camera.video is not None: + return {FileFormat.AV1_MP4.value} + return {FileFormat.JPG.value, FileFormat.PNG.value} + if sensor.lidar is not None: + return LIDAR_FRAME_FORMATS[sensor.lidar.container.format] + return {fileformat.value for fileformat in FileFormat} + + +# Tasks whose labels are checked against `capabilities.tasks`. +CHECKED_TASKS = ( + Task.DETECTION3D, + Task.DETECTION2D, + Task.SEGMENTATION2D, + Task.SEGMENTATION3D, + Task.KEYPOINT, + Task.TRAFFIC_LIGHT, +) + + +def declared_capabilities(context: SanityContext) -> set[str]: + """Return what `manifest.json` `capabilities` declares, as items like "camera:CAM_FRONT", "map", + "task:detection3d".""" + capabilities = context.manifest.unwrap().capabilities + items = { + f"{modality}:{channel}" + for modality in ("camera", "lidar", "radar") + for channel in getattr(capabilities.modalities, modality) + } + items |= { + name + for name in ("map", "route", "traffic_light", "vehicle_state", "scenario") + if getattr(capabilities, name) + } + items |= {f"task:{task.value}" for task in capabilities.tasks if task in CHECKED_TASKS} + return items + + +def present_capabilities(context: SanityContext) -> set[str]: + """Return what the dataset contains, as the same items as `declared_capabilities`.""" + + def records(table: V1SchemaName) -> list: + return context.load_records(table).unwrap() + + sensors = records(V1SchemaName.SENSOR) + with_frames = {frame.sensor_token for frame in records(V1SchemaName.SAMPLE_DATA)} + items = { + f"{sensor.modality.value}:{sensor.channel}" + for sensor in sensors + if sensor.token in with_frames + } + tables = { + "map": V1SchemaName.MAP, + "route": V1SchemaName.ROUTE, + "traffic_light": V1SchemaName.TRAFFIC_LIGHT, + "vehicle_state": V1SchemaName.VEHICLE_STATE, + "scenario": V1SchemaName.SCENARIO, + } + items |= {name for name, table in tables.items() if records(table)} + + things = {category.token for category in records(V1SchemaName.CATEGORY) if category.is_thing} + object_anns = records(V1SchemaName.OBJECT_ANN) + lidar_columns = { + field.name + for sensor in sensors + if sensor.modality == SensorModality.LIDAR and sensor.lidar is not None + for field in sensor.lidar.fields + } + tasks = { + Task.DETECTION3D: bool(records(V1SchemaName.SAMPLE_ANNOTATION)), + Task.DETECTION2D: any(ann.category_token in things for ann in object_anns), + Task.SEGMENTATION2D: any(ann.mask is not None for ann in object_anns), + Task.SEGMENTATION3D: "label" in lidar_columns, + Task.KEYPOINT: bool(records(V1SchemaName.KEYPOINT)), + Task.TRAFFIC_LIGHT: bool(records(V1SchemaName.TRAFFIC_LIGHT)) + or any(instance.map_element.id is not None for instance in records(V1SchemaName.INSTANCE)), + } + items |= {f"task:{task.value}" for task, present in tasks.items() if present} + return items diff --git a/t4_devkit/sanity/record/rec001.py b/t4_devkit/sanity/record/rec001.py index dff52a88..55dfeb21 100644 --- a/t4_devkit/sanity/record/rec001.py +++ b/t4_devkit/sanity/record/rec001.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC001(RecordCountChecker): id = RuleID("REC001") name = RuleName("scene-single") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene' record is a single." schema = SchemaName.SCENE diff --git a/t4_devkit/sanity/record/rec002.py b/t4_devkit/sanity/record/rec002.py index 48c66833..503dfa82 100644 --- a/t4_devkit/sanity/record/rec002.py +++ b/t4_devkit/sanity/record/rec002.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC002(RecordCountChecker): id = RuleID("REC002") name = RuleName("sample-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample' record is not empty." schema = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/record/rec003.py b/t4_devkit/sanity/record/rec003.py index 748f018d..e1b8f881 100644 --- a/t4_devkit/sanity/record/rec003.py +++ b/t4_devkit/sanity/record/rec003.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC003(RecordCountChecker): id = RuleID("REC003") name = RuleName("sample-data-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData' record is not empty." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/record/rec004.py b/t4_devkit/sanity/record/rec004.py index 93ac1cad..bdb71612 100644 --- a/t4_devkit/sanity/record/rec004.py +++ b/t4_devkit/sanity/record/rec004.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker @@ -17,6 +17,7 @@ class REC004(RecordCountChecker): id = RuleID("REC004") name = RuleName("ego-pose-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'EgoPose' record is not empty." schema = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/record/rec006.py b/t4_devkit/sanity/record/rec006.py index 1ed9ecc9..09ce9183 100644 --- a/t4_devkit/sanity/record/rec006.py +++ b/t4_devkit/sanity/record/rec006.py @@ -6,7 +6,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -25,6 +25,7 @@ class REC006(RecordCountChecker): id = RuleID("REC006") name = RuleName("instance-not-empty") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = ( "'Instance' record is not empty if either 'SampleAnnotation' or 'ObjectAnn' is not empty." ) @@ -45,8 +46,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case (None, _) | (_, None): return Maybe.from_value(Reason("Missing 'annotation' directory")) case (s, o): - sample_ann_count = len(load_json_safe(s).unwrap()) if s.exists() else 0 - object_ann_count = len(load_json_safe(o).unwrap()) if o.exists() else 0 + sample_ann_count = len(load_json_safe(s).value_or([])) if s.exists() else 0 + object_ann_count = len(load_json_safe(o).value_or([])) if o.exists() else 0 return ( Maybe.from_value( Reason( diff --git a/t4_devkit/sanity/record/rec007.py b/t4_devkit/sanity/record/rec007.py index 61a88d20..c9deb330 100644 --- a/t4_devkit/sanity/record/rec007.py +++ b/t4_devkit/sanity/record/rec007.py @@ -3,10 +3,11 @@ from typing import TYPE_CHECKING from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful from t4_devkit.schema import SchemaName -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -24,6 +25,7 @@ class REC007(Checker): id = RuleID("REC007") name = RuleName("category-indices-consistent") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "All categories must either have a unique 'index' or all have a 'null' index." schema = SchemaName.CATEGORY @@ -32,6 +34,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case Some(x): if not x.exists(): return Maybe.from_value(Reason(f"Missing {self.schema.filename}")) + elif not is_successful(load_json_safe(x)): + return Maybe.from_value(Reason(f"Cannot read {self.schema.filename}")) else: return Nothing case _: diff --git a/t4_devkit/sanity/record/rec008.py b/t4_devkit/sanity/record/rec008.py new file mode 100644 index 00000000..62c42a36 --- /dev/null +++ b/t4_devkit/sanity/record/rec008.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC008"] + + +@CHECKERS.register() +class REC008(RecordsChecker): + """A checker of REC008.""" + + id = RuleID("REC008") + name = RuleName("sensor-not-empty") + severity = Severity.ERROR + description = "'Sensor' record is not empty." + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + if not context.load_records(SchemaName.SENSOR).unwrap(): + return [Reason("'Sensor' record must not be empty")] + return None diff --git a/t4_devkit/sanity/record/rec009.py b/t4_devkit/sanity/record/rec009.py new file mode 100644 index 00000000..67beca3a --- /dev/null +++ b/t4_devkit/sanity/record/rec009.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import Counter + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC009"] + + +@CHECKERS.register() +class REC009(RecordsChecker): + """A checker of REC009.""" + + id = RuleID("REC009") + name = RuleName("sensor-index-unique") + severity = Severity.ERROR + description = "'Sensor.index' values are unique." + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + counts = Counter( + sensor.index for sensor in context.load_records(SchemaName.SENSOR).unwrap() + ) + return [ + Reason(f"'Sensor.index' {index} is used by {count} sensors") + for index, count in sorted(counts.items()) + if count > 1 + ] or None diff --git a/t4_devkit/sanity/record/rec010.py b/t4_devkit/sanity/record/rec010.py new file mode 100644 index 00000000..f997ab8d --- /dev/null +++ b/t4_devkit/sanity/record/rec010.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import Counter + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC010"] + + +@CHECKERS.register() +class REC010(RecordsChecker): + """A checker of REC010.""" + + id = RuleID("REC010") + name = RuleName("instance-index-unique") + severity = Severity.ERROR + description = "'Instance.index' values are unique where set." + tables = (SchemaName.INSTANCE,) + + def check(self, context: SanityContext) -> list[Reason] | None: + counts = Counter( + instance.index + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + if instance.index is not None + ) + return [ + Reason(f"'Instance.index' {index} is used by {count} instances") + for index, count in sorted(counts.items()) + if count > 1 + ] or None diff --git a/t4_devkit/sanity/record/rec011.py b/t4_devkit/sanity/record/rec011.py new file mode 100644 index 00000000..4b033a41 --- /dev/null +++ b/t4_devkit/sanity/record/rec011.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC011"] + + +@CHECKERS.register() +class REC011(RecordsChecker): + """A checker of REC011.""" + + id = RuleID("REC011") + name = RuleName("point-label-indices-presence") + severity = Severity.ERROR + description = ( + "When a LiDAR file has 'label' / 'instance' columns, every 'Category' has an 'index', " + "and every 'Instance' of an 'is_thing' category an 'index'." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.CATEGORY, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + columns = { + field.name + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + if sensor.lidar is not None + for field in sensor.lidar.fields + } + categories = context.load_records(SchemaName.CATEGORY).unwrap() + failures = [] + if "label" in columns: + failures += [ + Reason(f"Category without 'index': {category.name}") + for category in categories + if category.index is None + ] + if "instance" in columns: + things = {category.token for category in categories if category.is_thing} + failures += [ + Reason(f"Instance without 'index': {instance.token}") + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + if instance.category_token in things and instance.index is None + ] + return failures or None diff --git a/t4_devkit/sanity/record/rec012.py b/t4_devkit/sanity/record/rec012.py new file mode 100644 index 00000000..ae0c2485 --- /dev/null +++ b/t4_devkit/sanity/record/rec012.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import Counter + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC012"] + + +@CHECKERS.register() +class REC012(RecordsChecker): + """A checker of REC012.""" + + id = RuleID("REC012") + name = RuleName("scene-sample-count") + severity = Severity.ERROR + description = "'Scene.nbr_samples' equals the number of 'Sample' records of the scene." + tables = ( + SchemaName.SCENE, + SchemaName.SAMPLE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + counts = _sample_counts(context) + return [ + Reason( + f"Scene {scene.token}: 'nbr_samples' {scene.nbr_samples}, has {counts[scene.token]}" + ) + for scene in context.load_records(SchemaName.SCENE).unwrap() + if scene.nbr_samples != counts[scene.token] + ] or None + + def fix(self, context: SanityContext) -> bool: + counts = _sample_counts(context) + records = context.load_table(SchemaName.SCENE).unwrap() + for record in records: + record["nbr_samples"] = counts[record["token"]] + return context.save_table(SchemaName.SCENE, records) + + +def _sample_counts(context: SanityContext) -> Counter[str]: + return Counter( + sample.scene_token for sample in context.load_records(SchemaName.SAMPLE).unwrap() + ) diff --git a/t4_devkit/sanity/record/rec013.py b/t4_devkit/sanity/record/rec013.py new file mode 100644 index 00000000..8568f388 --- /dev/null +++ b/t4_devkit/sanity/record/rec013.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC013"] + + +@CHECKERS.register() +class REC013(RecordsChecker): + """A checker of REC013.""" + + id = RuleID("REC013") + name = RuleName("instance-annotation-count") + severity = Severity.ERROR + description = ( + "'Instance.nbr_annotations' equals the number of its 'SampleAnnotation' and 'ObjectAnn' " + "records, and its first / last annotation tokens point to its first / last one in time." + ) + tables = ( + SchemaName.INSTANCE, + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + expected = _expected_annotations(context) + failures = [] + for instance in context.load_records(SchemaName.INSTANCE).unwrap(): + count, first, last = expected.get(instance.token, (0, None, None)) + if instance.nbr_annotations != count: + failures.append( + Reason( + f"Instance {instance.token}: 'nbr_annotations' {instance.nbr_annotations}, has {count}" + ) + ) + if (instance.first_annotation_token, instance.last_annotation_token) != (first, last): + failures.append( + Reason( + f"Instance {instance.token}: first / last annotation " + f"{instance.first_annotation_token} / {instance.last_annotation_token}, " + f"expected {first} / {last}" + ) + ) + return failures or None + + def fix(self, context: SanityContext) -> bool: + expected = _expected_annotations(context) + records = context.load_table(SchemaName.INSTANCE).unwrap() + for record in records: + count, first, last = expected.get(record["token"], (0, None, None)) + record["nbr_annotations"] = count + record["first_annotation_token"] = first + record["last_annotation_token"] = last + return context.save_table(SchemaName.INSTANCE, records) + + +def _expected_annotations(context: SanityContext) -> dict[str, tuple[int, str, str]]: + """Return the count, first and last annotation token in time of each instance with annotations. + + A `SampleAnnotation` is at its sample's time, an `ObjectAnn` at its frame's time. + """ + samples = {sample.token: sample for sample in context.load_records(SchemaName.SAMPLE).unwrap()} + frames = {frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap()} + times: dict[str, list[tuple[int, str]]] = defaultdict(list) + for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap(): + if ann.sample_token in samples: + times[ann.instance_token].append((samples[ann.sample_token].timestamp, ann.token)) + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap(): + if ann.instance_token is not None and ann.sample_data_token in frames: + times[ann.instance_token].append((frames[ann.sample_data_token].timestamp, ann.token)) + return { + token: (len(entries), min(entries)[1], max(entries)[1]) for token, entries in times.items() + } diff --git a/t4_devkit/sanity/record/rec014.py b/t4_devkit/sanity/record/rec014.py new file mode 100644 index 00000000..c831de74 --- /dev/null +++ b/t4_devkit/sanity/record/rec014.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC014"] + + +@CHECKERS.register() +class REC014(RecordsChecker): + """A checker of REC014.""" + + id = RuleID("REC014") + name = RuleName("thing-category-instance") + severity = Severity.ERROR + description = ( + "'ObjectAnn.instance_token' is set exactly when its category has 'is_thing'; " + "'SampleAnnotation' and 'Keypoint' instances belong to 'is_thing' categories." + ) + tables = ( + SchemaName.CATEGORY, + SchemaName.INSTANCE, + SchemaName.OBJECT_ANN, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.KEYPOINT, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + is_thing = { + category.token: category.is_thing + for category in context.load_records(SchemaName.CATEGORY).unwrap() + } + instance_category = { + instance.token: instance.category_token + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + } + failures = [] + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap(): + thing = is_thing.get(ann.category_token) + if thing is not None and thing != (ann.instance_token is not None): + expected = "an instance" if thing else "no instance" + failures.append(Reason(f"ObjectAnn {ann.token}: its category needs {expected}")) + for table in (SchemaName.SAMPLE_ANNOTATION, SchemaName.KEYPOINT): + for ann in context.load_records(table).unwrap(): + category = instance_category.get(ann.instance_token) + if category in is_thing and not is_thing[category]: + failures.append( + Reason( + f"{record_name(table)} {ann.token}: instance of a category without 'is_thing'" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec015.py b/t4_devkit/sanity/record/rec015.py new file mode 100644 index 00000000..53b58de2 --- /dev/null +++ b/t4_devkit/sanity/record/rec015.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC015"] + + +@CHECKERS.register() +class REC015(RecordsChecker): + """A checker of REC015.""" + + id = RuleID("REC015") + name = RuleName("orientation-number-allowed") + severity = Severity.ERROR + description = ( + "'ObjectAnn.orientation' / 'number' are set only for categories with " + "'has_orientation' / 'has_number'." + ) + tables = ( + SchemaName.CATEGORY, + SchemaName.OBJECT_ANN, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + categories = { + category.token: category + for category in context.load_records(SchemaName.CATEGORY).unwrap() + } + failures = [] + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap(): + category = categories.get(ann.category_token) + if category is None: + continue # reported by the reference rules + if ann.orientation is not None and not category.has_orientation: + failures.append(Reason(f"ObjectAnn {ann.token}: 'orientation' for {category.name}")) + if ann.number is not None and not category.has_number: + failures.append(Reason(f"ObjectAnn {ann.token}: 'number' for {category.name}")) + return failures or None diff --git a/t4_devkit/sanity/record/rec016.py b/t4_devkit/sanity/record/rec016.py new file mode 100644 index 00000000..df512b3d --- /dev/null +++ b/t4_devkit/sanity/record/rec016.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import AnnotationType, SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC016"] + + +@CHECKERS.register() +class REC016(RecordsChecker): + """A checker of REC016.""" + + id = RuleID("REC016") + name = RuleName("manual-label-confidence") + severity = Severity.ERROR + description = "Labels with 'annotation_type: manual' have 'confidence: null'." + tables = ( + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.KEYPOINT, + SchemaName.TRAFFIC_LIGHT, + SchemaName.SCENARIO, + SchemaName.CAPTION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return [ + Reason( + f"{record_name(table)} {label.token}: manual label with 'confidence' {label.confidence}" + ) + for table in self.tables + for label in context.load_records(table).unwrap() + if label.annotation_type == AnnotationType.MANUAL + and getattr(label, "confidence", None) is not None + ] or None diff --git a/t4_devkit/sanity/record/rec017.py b/t4_devkit/sanity/record/rec017.py new file mode 100644 index 00000000..6454cafb --- /dev/null +++ b/t4_devkit/sanity/record/rec017.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import AnnotationType, SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC017"] + + +@CHECKERS.register() +class REC017(RecordsChecker): + """A checker of REC017.""" + + id = RuleID("REC017") + name = RuleName("automatic-label-provenance") + severity = Severity.WARNING + description = ( + "Labels that are not 'manual' have a 'provenance', and 'auto' / 'pseudo' labels a " + "'confidence'." + ) + tables = ( + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.KEYPOINT, + SchemaName.TRAFFIC_LIGHT, + SchemaName.SCENARIO, + SchemaName.CAPTION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + for table in self.tables: + for label in context.load_records(table).unwrap(): + if label.annotation_type == AnnotationType.MANUAL: + continue + if label.provenance is None: + failures.append(Reason(f"{record_name(table)} {label.token}: no 'provenance'")) + scored = label.annotation_type in (AnnotationType.AUTO, AnnotationType.PSEUDO) + # `traffic_light` has no record-level confidence + if scored and hasattr(label, "confidence") and label.confidence is None: + failures.append(Reason(f"{record_name(table)} {label.token}: no 'confidence'")) + return failures or None diff --git a/t4_devkit/sanity/record/rec018.py b/t4_devkit/sanity/record/rec018.py new file mode 100644 index 00000000..9f790534 --- /dev/null +++ b/t4_devkit/sanity/record/rec018.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING + +from t4_devkit.schema import SensorModality +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC018"] + + +@CHECKERS.register() +class REC018(RecordsChecker): + """A checker of REC018.""" + + id = RuleID("REC018") + name = RuleName("sample-frames") + severity = Severity.ERROR + description = ( + "A 'Sample' has at most one 'SampleData' per channel; its 'timestamp' is its LiDAR frame's " + "(its earliest frame's if it has no LiDAR); 'has_key_frame' is true exactly when one of " + "its 'SampleData' is a key frame." + ) + tables = (SchemaName.SAMPLE, SchemaName.SAMPLE_DATA, SchemaName.SENSOR) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = { + sensor.token: sensor for sensor in context.load_records(SchemaName.SENSOR).unwrap() + } + frames: dict[str, list] = defaultdict(list) + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + if frame.sensor_token in sensors: # unknown sensors: reported by the reference rules + frames[frame.sample_token].append(frame) + + failures = [] + for sample in context.load_records(SchemaName.SAMPLE).unwrap(): + sample_frames = frames[sample.token] + channels = [sensors[frame.sensor_token].channel for frame in sample_frames] + for channel in sorted({c for c in channels if channels.count(c) > 1}): + failures.append(Reason(f"Sample {sample.token}: several frames of {channel}")) + if sample_frames: + lidar_frames = [ + frame + for frame in sample_frames + if sensors[frame.sensor_token].modality == SensorModality.LIDAR + ] + expected = min(frame.timestamp for frame in lidar_frames or sample_frames) + if sample.timestamp != expected: + failures.append( + Reason( + f"Sample {sample.token}: 'timestamp' {sample.timestamp}, expected {expected}" + ) + ) + has_key_frame = any(frame.is_key_frame for frame in sample_frames) + if sample.has_key_frame != has_key_frame: + failures.append( + Reason(f"Sample {sample.token}: 'has_key_frame' should be {has_key_frame}") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec019.py b/t4_devkit/sanity/record/rec019.py new file mode 100644 index 00000000..c59ac5b3 --- /dev/null +++ b/t4_devkit/sanity/record/rec019.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from .base import channel_file + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC019"] + + +@CHECKERS.register() +class REC019(RecordsChecker): + """A checker of REC019.""" + + id = RuleID("REC019") + name = RuleName("channel-frames") + severity = Severity.ERROR + description = ( + "For a channel stored in one file, 'frame_count' equals the number of its 'SampleData' " + "records, and their 'frame_index' values are 0 to 'frame_count - 1', each once." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = context.load_records(SchemaName.SENSOR).unwrap() + # source LiDARs of a merged channel have no frames of their own + sources = { + channel + for sensor in sensors + if sensor.lidar is not None + for channel in sensor.lidar.source_channels + } + indices: dict[str, list[int | None]] = defaultdict(list) + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + indices[frame.sensor_token].append(frame.frame_index) + failures = [] + for sensor in sensors: + stored = channel_file(sensor) + if stored is None or sensor.channel in sources: + continue + _, frame_count = stored + if sorted(indices[sensor.token], key=lambda i: -1 if i is None else i) != list( + range(frame_count) + ): + failures.append( + Reason( + f"Sensor {sensor.channel} ({sensor.token}): 'frame_count' {frame_count}, " + f"{len(indices[sensor.token])} frames, 'frame_index' not 0 to {frame_count - 1}" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec020.py b/t4_devkit/sanity/record/rec020.py new file mode 100644 index 00000000..183f1de7 --- /dev/null +++ b/t4_devkit/sanity/record/rec020.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from .base import channel_file, frame_formats + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC020"] + + +@CHECKERS.register() +class REC020(RecordsChecker): + """A checker of REC020.""" + + id = RuleID("REC020") + name = RuleName("sensor-storage-consistent") + severity = Severity.ERROR + description = ( + "'SampleData.frame_index' is 'null' exactly when its sensor stores one file per frame, " + "and 'fileformat' matches the sensor's storage." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = { + sensor.token: sensor for sensor in context.load_records(SchemaName.SENSOR).unwrap() + } + failures = [] + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + sensor = sensors.get(frame.sensor_token) + if sensor is None: + continue # reported by the reference rules + one_file = channel_file(sensor) is not None + if one_file != (frame.frame_index is not None): + expected = "a 'frame_index'" if one_file else "no 'frame_index'" + failures.append(Reason(f"SampleData {frame.token}: needs {expected}")) + formats = frame_formats(sensor) + if frame.fileformat.value not in formats: + failures.append( + Reason( + f"SampleData {frame.token}: 'fileformat' {frame.fileformat.value}, " + f"sensor {sensor.channel} stores {' / '.join(sorted(formats))}" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec021.py b/t4_devkit/sanity/record/rec021.py new file mode 100644 index 00000000..0e88fcc6 --- /dev/null +++ b/t4_devkit/sanity/record/rec021.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC021"] + + +@CHECKERS.register() +class REC021(RecordsChecker): + """A checker of REC021.""" + + id = RuleID("REC021") + name = RuleName("timestamps-ordered") + severity = Severity.ERROR + description = ( + "Timestamps increase along 'next' ('Sample', 'SampleData', 'SampleAnnotation'), and " + "'frame_index' follows timestamp order within a channel file." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + samples = { + sample.token: sample for sample in context.load_records(SchemaName.SAMPLE).unwrap() + } + frames = { + frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + } + anns = { + ann.token: ann for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + } + + def ann_time(token: str) -> int | None: + sample = samples.get(anns[token].sample_token) + return None if sample is None else sample.timestamp + + failures = [] + for name, records, time in ( + ("Sample", samples, lambda token: samples[token].timestamp), + ("SampleData", frames, lambda token: frames[token].timestamp), + ("SampleAnnotation", anns, ann_time), + ): + for token, record in records.items(): + if record.next in records: + current, following = time(token), time(record.next) + if current is not None and following is not None and following <= current: + failures.append(Reason(f"{name} {token}: 'next' is not later")) + + by_sensor: dict[str, list] = defaultdict(list) + for frame in frames.values(): + if frame.frame_index is not None: + by_sensor[frame.sensor_token].append(frame) + for sensor_token, sensor_frames in by_sensor.items(): + ordered = sorted(sensor_frames, key=lambda frame: frame.frame_index) + if any(a.timestamp >= b.timestamp for a, b in zip(ordered, ordered[1:])): + failures.append( + Reason(f"Sensor {sensor_token}: 'frame_index' not in timestamp order") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec022.py b/t4_devkit/sanity/record/rec022.py new file mode 100644 index 00000000..fd1f135d --- /dev/null +++ b/t4_devkit/sanity/record/rec022.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC022"] + + +@CHECKERS.register() +class REC022(RecordsChecker): + """A checker of REC022.""" + + id = RuleID("REC022") + name = RuleName("timestamps-microseconds") + severity = Severity.WARNING + description = ( + "Timestamps are Unix microseconds: between 2000-01-01 and 2100-01-01 (catches seconds or " + "nanoseconds)." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.EGO_POSE, + SchemaName.VEHICLE_STATE, + SchemaName.TRAFFIC_LIGHT, + SchemaName.ROUTE, + SchemaName.SCENARIO, + SchemaName.CAPTION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + for table, fields in TIMESTAMP_FIELDS.items(): + for record in context.load_records(table).unwrap(): + for name in fields: + value = getattr(record, name) + if value is not None and not MIN_TIMESTAMP <= value < MAX_TIMESTAMP: + failures.append( + Reason(f"{record_name(table)} {record.token}: '{name}' {value}") + ) + return failures or None + + +# 2000-01-01 and 2100-01-01 in Unix microseconds. +MIN_TIMESTAMP = 946_684_800_000_000 +MAX_TIMESTAMP = 4_102_444_800_000_000 + +TIMESTAMP_FIELDS = { + SchemaName.SAMPLE: ("timestamp",), + SchemaName.SAMPLE_DATA: ("timestamp",), + SchemaName.EGO_POSE: ("timestamp",), + SchemaName.VEHICLE_STATE: ("timestamp",), + SchemaName.TRAFFIC_LIGHT: ("timestamp",), + SchemaName.ROUTE: ("valid_from", "valid_until"), + SchemaName.SCENARIO: ("start_timestamp", "end_timestamp", "key_timestamp"), + SchemaName.CAPTION: ("start_timestamp", "end_timestamp"), +} diff --git a/t4_devkit/sanity/record/rec023.py b/t4_devkit/sanity/record/rec023.py new file mode 100644 index 00000000..3ea679fc --- /dev/null +++ b/t4_devkit/sanity/record/rec023.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from .base import declared_capabilities, present_capabilities + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC023"] + + +@CHECKERS.register() +class REC023(RecordsChecker): + """A checker of REC023.""" + + id = RuleID("REC023") + name = RuleName("capabilities-present") + severity = Severity.ERROR + description = ( + "Everything 'manifest.json' 'capabilities' declares is present: channels, map, tables and " + "the labels of its tasks." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + SchemaName.MAP, + SchemaName.ROUTE, + SchemaName.TRAFFIC_LIGHT, + SchemaName.VEHICLE_STATE, + SchemaName.SCENARIO, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.CATEGORY, + SchemaName.KEYPOINT, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + present = present_capabilities(context) + return [ + Reason(f"Declared but not present: {item}") + for item in sorted(declared_capabilities(context) - present) + ] or None diff --git a/t4_devkit/sanity/record/rec024.py b/t4_devkit/sanity/record/rec024.py new file mode 100644 index 00000000..2c4ca181 --- /dev/null +++ b/t4_devkit/sanity/record/rec024.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from .base import declared_capabilities, present_capabilities + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC024"] + + +@CHECKERS.register() +class REC024(RecordsChecker): + """A checker of REC024.""" + + id = RuleID("REC024") + name = RuleName("capabilities-complete") + severity = Severity.WARNING + description = ( + "Everything present is declared in 'capabilities': the same checks as REC023, the other " + "way round." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + SchemaName.MAP, + SchemaName.ROUTE, + SchemaName.TRAFFIC_LIGHT, + SchemaName.VEHICLE_STATE, + SchemaName.SCENARIO, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.CATEGORY, + SchemaName.KEYPOINT, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + declared = declared_capabilities(context) + return [ + Reason(f"Present but not declared: {item}") + for item in sorted(present_capabilities(context) - declared) + ] or None diff --git a/t4_devkit/sanity/record/rec025.py b/t4_devkit/sanity/record/rec025.py new file mode 100644 index 00000000..93967873 --- /dev/null +++ b/t4_devkit/sanity/record/rec025.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName, V0Converter + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC025"] + + +@CHECKERS.register() +class REC025(RecordsChecker): + """A checker of REC025.""" + + id = RuleID("REC025") + name = RuleName("no-placeholder-values") + severity = Severity.WARNING + description = ( + "No placeholder values from loading or migrating 0.x: 'vehicle.model' and 'map.map_id' / " + "'version_id' are not 'unknown'." + ) + tables = ( + SchemaName.VEHICLE, + SchemaName.MAP, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + vehicle_model = V0Converter.FALLBACKS["vehicle"]["model"] + map_fallbacks = V0Converter.FALLBACKS["map"] + failures = [ + Reason(f"Vehicle {vehicle.token}: placeholder 'model' {vehicle.model!r}") + for vehicle in context.load_records(SchemaName.VEHICLE).unwrap() + if vehicle.model == vehicle_model + ] + for map_record in context.load_records(SchemaName.MAP).unwrap(): + for name in ("map_id", "version_id"): + value = getattr(map_record, name) + if value == map_fallbacks[name]: + failures.append( + Reason(f"Map {map_record.token}: placeholder '{name}' {value!r}") + ) + return failures or None diff --git a/t4_devkit/sanity/record/rec026.py b/t4_devkit/sanity/record/rec026.py new file mode 100644 index 00000000..b886bb15 --- /dev/null +++ b/t4_devkit/sanity/record/rec026.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from collections import defaultdict + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC026"] + + +@CHECKERS.register() +class REC026(RecordsChecker): + """A checker of REC026.""" + + id = RuleID("REC026") + name = RuleName("route-validity") + severity = Severity.ERROR + description = ( + "'route.valid_until' is after 'valid_from' when set, and the routes of a log do not overlap " + "in time." + ) + tables = (SchemaName.ROUTE,) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + by_log: dict[str, list] = defaultdict(list) + for route in context.load_records(SchemaName.ROUTE).unwrap(): + if route.valid_until is not None and route.valid_until <= route.valid_from: + failures.append( + Reason(f"Route {route.token}: 'valid_until' not after 'valid_from'") + ) + by_log[route.log_token].append(route) + for routes in by_log.values(): + ordered = sorted(routes, key=lambda route: route.valid_from) + for route, following in zip(ordered, ordered[1:]): + if route.valid_until is None or route.valid_until > following.valid_from: + failures.append(Reason(f"Routes {route.token} and {following.token} overlap")) + return failures or None diff --git a/t4_devkit/sanity/record/rec027.py b/t4_devkit/sanity/record/rec027.py new file mode 100644 index 00000000..929296ef --- /dev/null +++ b/t4_devkit/sanity/record/rec027.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC027"] + + +@CHECKERS.register() +class REC027(RecordsChecker): + """A checker of REC027.""" + + id = RuleID("REC027") + name = RuleName("unit-quaternions") + severity = Severity.WARNING + description = ( + "Rotations ('Sensor', 'EgoPose', 'SampleAnnotation', 'Route' poses) are unit quaternions." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.EGO_POSE, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.ROUTE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + rotations = [ + (record_name(table), record.token, record.rotation) + for table in (SchemaName.SENSOR, SchemaName.EGO_POSE, SchemaName.SAMPLE_ANNOTATION) + for record in context.load_records(table).unwrap() + ] + [ + (record_name(SchemaName.ROUTE), route.token, pose.rotation) + for route in context.load_records(SchemaName.ROUTE).unwrap() + for pose in (route.start_pose, route.goal_pose) + ] + return [ + Reason(f"{table} {token}: rotation norm {rotation.norm:.6f}") + for table, token, rotation in rotations + if abs(rotation.norm - 1.0) > NORM_TOLERANCE + ] or None + + +NORM_TOLERANCE = 1e-3 diff --git a/t4_devkit/sanity/record/rec028.py b/t4_devkit/sanity/record/rec028.py new file mode 100644 index 00000000..2e53bdfd --- /dev/null +++ b/t4_devkit/sanity/record/rec028.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC028"] + + +@CHECKERS.register() +class REC028(RecordsChecker): + """A checker of REC028.""" + + id = RuleID("REC028") + name = RuleName("labels-on-key-frames") + severity = Severity.ERROR + description = ( + "Labels sit on key frames: a 'SampleAnnotation' on a 'Sample' with 'has_key_frame', an " + "'ObjectAnn' or 'Keypoint' on a 'SampleData' with 'is_key_frame'." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.OBJECT_ANN, + SchemaName.KEYPOINT, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + key_samples = { + sample.token + for sample in context.load_records(SchemaName.SAMPLE).unwrap() + if sample.has_key_frame + } + key_frames = { + frame.token + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + if frame.is_key_frame + } + failures = [ + Reason(f"SampleAnnotation {ann.token}: sample without a key frame") + for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + if ann.sample_token not in key_samples + ] + for table in (SchemaName.OBJECT_ANN, SchemaName.KEYPOINT): + failures += [ + Reason(f"{record_name(table)} {ann.token}: not on a key frame") + for ann in context.load_records(table).unwrap() + if ann.sample_data_token not in key_frames + ] + return failures or None diff --git a/t4_devkit/sanity/record/rec029.py b/t4_devkit/sanity/record/rec029.py new file mode 100644 index 00000000..c313fd89 --- /dev/null +++ b/t4_devkit/sanity/record/rec029.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..reference.files import lidar_partly_null_frames +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REC029"] + + +@CHECKERS.register() +class REC029(RecordsChecker): + """A checker of REC029.""" + + id = RuleID("REC029") + name = RuleName("point-labels-reserved") + severity = Severity.ERROR + description = ( + "When a LiDAR file has 'label' / 'instance' columns: the 'Category' with 'index' 0 is " + "named 'unpainted', and each frame's label columns are null on all points or on none." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.CATEGORY, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = [s for s in context.load_records(SchemaName.SENSOR).unwrap() if s.lidar] + columns = {field.name for sensor in sensors for field in sensor.lidar.fields} + failures = [] + if "label" in columns: + names = [ + c.name for c in context.load_records(SchemaName.CATEGORY).unwrap() if c.index == 0 + ] + if not names: + failures.append(Reason("No 'Category' with 'index' 0 ('unpainted')")) + elif names != ["unpainted"]: + failures.append(Reason(f"'Category' with 'index' 0 is {names}, not 'unpainted'")) + + data_root = context.data_root.unwrap() + for sensor in sensors: + if sensor.lidar.container.path is None: + continue + path = data_root.joinpath(sensor.lidar.container.path) + if not path.exists(): + continue + for column in ("label", "instance"): + frames = lidar_partly_null_frames(path, column) + if frames: + failures.append( + Reason( + f"Sensor {sensor.channel}: '{column}' is null on only some points of " + f"frames {frames[:10]}" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/reference/__init__.py b/t4_devkit/sanity/reference/__init__.py index b9fbaca5..74adcd09 100644 --- a/t4_devkit/sanity/reference/__init__.py +++ b/t4_devkit/sanity/reference/__init__.py @@ -24,3 +24,25 @@ from .ref202 import * # noqa from .ref203 import * # noqa from .ref301 import * # noqa +from .ref015 import * # noqa +from .ref016 import * # noqa +from .ref017 import * # noqa +from .ref018 import * # noqa +from .ref019 import * # noqa +from .ref020 import * # noqa +from .ref021 import * # noqa +from .ref022 import * # noqa +from .ref023 import * # noqa +from .ref024 import * # noqa +from .ref025 import * # noqa +from .ref107 import * # noqa +from .ref108 import * # noqa +from .ref204 import * # noqa +from .ref205 import * # noqa +from .ref206 import * # noqa +from .ref207 import * # noqa +from .ref302 import * # noqa +from .ref303 import * # noqa +from .ref304 import * # noqa +from .ref305 import * # noqa +from .ref306 import * # noqa diff --git a/t4_devkit/sanity/reference/base.py b/t4_devkit/sanity/reference/base.py index 5b32f76e..961b7c14 100644 --- a/t4_devkit/sanity/reference/base.py +++ b/t4_devkit/sanity/reference/base.py @@ -5,12 +5,13 @@ from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful -from ..checker import Checker +from ..checker import Checker, record_name from ..result import Reason from ..safety import load_json_safe if TYPE_CHECKING: from t4_devkit.schema import SchemaName + from t4_devkit.schema.v1 import SchemaName as V1SchemaName from ..context import SanityContext @@ -41,6 +42,8 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: target_file_paths = [y.unwrap() for y in ys if is_successful(y)] if not x.exists(): return Maybe.from_value(Reason(f"Missing {self.source.filename}")) + elif not is_successful(load_json_safe(x)): + return Maybe.from_value(Reason(f"Cannot read {self.source.filename}")) elif not any(path.exists() for path in target_file_paths): return Maybe.from_value( Reason(f"Missing {', '.join(t.filename for t in target_schemas)}") @@ -59,18 +62,19 @@ def check(self, context: SanityContext) -> list[Reason] | None: if is_successful(target_file) and target_file.unwrap().exists() ] source_records = load_json_safe(source_file).unwrap() - target_tokens = [ + target_tokens = { item["token"] for target_file in target_files - for item in load_json_safe(target_file).unwrap() - ] + for item in load_json_safe(target_file).value_or([]) + } return [ Reason( f"No reference to '{self.source.value}.{self.reference}': {record[self.reference]}" ) for record in source_records if record[self.reference] not in target_tokens - and record[self.reference] != "" # NOTE: success if the reference token is "" + # NOTE: success if the reference token is "" (0.x) or null (1.0) + and record[self.reference] not in ("", None) and self.is_additional_condition_ok(record) ] or None @@ -120,3 +124,39 @@ class ExternalReferenceChecker(Checker): target: SchemaName reference: str + + +def missing_references( + context: SanityContext, + source: V1SchemaName, + fields: dict[str, V1SchemaName], +) -> list[Reason]: + """Return the 1.0 links of a table that refer to no record: one reason per missing token. + + Args: + context (SanityContext): The sanity context. + source (V1SchemaName): The table whose records hold the links. + fields (dict[str, V1SchemaName]): Link field (a token, a list of tokens, or null for no + link) and the table it refers to. + + Returns: + A reason per link to a missing record. + """ + tokens = { + table: {record.token for record in context.load_records(table).unwrap()} + for table in set(fields.values()) + } + reasons = [] + for record in context.load_records(source).unwrap(): + for name, table in fields.items(): + value = getattr(record, name) + values = value if isinstance(value, list) else [value] + reasons += [ + Reason( + f"{record_name(source)} {record.token}: '{name}' {token} refers to no " + f"{record_name(table)}" + ) + for token in values + if token is not None and token not in tokens[table] + ] + return reasons diff --git a/t4_devkit/sanity/reference/files.py b/t4_devkit/sanity/reference/files.py new file mode 100644 index 00000000..dd66cc8e --- /dev/null +++ b/t4_devkit/sanity/reference/files.py @@ -0,0 +1,100 @@ +"""Reading 1.0 channel files for the file reference rules.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import av +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from ..result import Reason + +if TYPE_CHECKING: + from pathlib import Path + + from t4_devkit.schema.v1 import Sensor + +__all__ = ["describe_lidar_file", "describe_video_file", "lidar_column_values"] + + +def describe_lidar_file(sensor: Sensor, path: Path) -> list[Reason]: + """Return the differences between a LiDAR Parquet file and its sensor row (columns, dtypes, + frames).""" + parquet = pq.ParquetFile(path) + columns = [(field.name, _numpy_dtype(field.type)) for field in parquet.schema_arrow] + declared = [(field.name, field.dtype) for field in sensor.lidar.fields] + failures = [] + if columns != declared: + failures.append(Reason(f"Sensor {sensor.channel}: columns {columns}, declared {declared}")) + if parquet.num_row_groups != sensor.lidar.container.frame_count: + failures.append( + Reason( + f"Sensor {sensor.channel}: {parquet.num_row_groups} row groups, " + f"'frame_count' {sensor.lidar.container.frame_count}" + ) + ) + return failures + + +def describe_video_file(sensor: Sensor, path: Path) -> list[Reason]: + """Return the differences between a camera video and its sensor row (codec, pixels, timing, + size).""" + camera, video = sensor.camera, sensor.camera.video + with av.open(str(path)) as container: + stream = container.streams.video[0] + frame = next(container.decode(stream)) + actual = { + "codec": stream.codec_context.codec.name, + "pixel_format": frame.format.name, + "bit_depth": frame.format.components[0].bits, + "timescale": stream.time_base.denominator, + "frame_count": stream.frames, + "size": (frame.width, frame.height), + } + expected = { + "codec": "av1", + "pixel_format": video.pixel_format.split()[0], + "bit_depth": video.bit_depth, + "timescale": video.timescale, + "frame_count": video.frame_count, + "size": (camera.width, camera.height), + } + # decoders of AV1 have their own names (libdav1d, libaom-av1, ...) + if "av1" in actual["codec"]: + actual["codec"] = "av1" + return [ + Reason(f"Sensor {sensor.channel}: {name} {actual[name]}, declared {expected[name]}") + for name in expected + if actual[name] != expected[name] + ] + + +def lidar_column_values(path: Path, column: str) -> set | None: + """Return the distinct non-null values of a column of a LiDAR Parquet file, or None if it has + no such column.""" + parquet = pq.ParquetFile(path) + if column not in parquet.schema_arrow.names: + return None + values = pc.unique(parquet.read(columns=[column]).column(column)).drop_null() + return set(values.to_pylist()) + + +def lidar_partly_null_frames(path: Path, column: str) -> list[int] | None: + """Return the frames of a LiDAR Parquet file where a column is null on some points but not + all, or None if it has no such column.""" + parquet = pq.ParquetFile(path) + if column not in parquet.schema_arrow.names: + return None + frames = [] + for i in range(parquet.num_row_groups): + nulls = parquet.read_row_group(i, columns=[column]).column(column).null_count + if 0 < nulls < parquet.metadata.row_group(i).num_rows: + frames.append(i) + return frames + + +def _numpy_dtype(arrow_type: pa.DataType) -> str: + """Return the NumPy name of an Arrow type, as `sensor.lidar.fields` writes it (e.g. "float32").""" + return pa.array([], type=arrow_type).to_numpy(zero_copy_only=False).dtype.name diff --git a/t4_devkit/sanity/reference/ref001.py b/t4_devkit/sanity/reference/ref001.py index c4f69e95..94f63422 100644 --- a/t4_devkit/sanity/reference/ref001.py +++ b/t4_devkit/sanity/reference/ref001.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF001(RecordReferenceChecker): id = RuleID("REF001") name = RuleName("scene-to-log") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene.log_token' refers to 'Log' record." source = SchemaName.SCENE target = SchemaName.LOG diff --git a/t4_devkit/sanity/reference/ref002.py b/t4_devkit/sanity/reference/ref002.py index 0ad3a835..1a38ee2c 100644 --- a/t4_devkit/sanity/reference/ref002.py +++ b/t4_devkit/sanity/reference/ref002.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF002(RecordReferenceChecker): id = RuleID("REF002") name = RuleName("scene-to-first-sample") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene.first_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref003.py b/t4_devkit/sanity/reference/ref003.py index 420d231c..1414118f 100644 --- a/t4_devkit/sanity/reference/ref003.py +++ b/t4_devkit/sanity/reference/ref003.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF003(RecordReferenceChecker): id = RuleID("REF003") name = RuleName("scene-to-last-sample") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Scene.last_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref004.py b/t4_devkit/sanity/reference/ref004.py index a8bea29a..533b1c65 100644 --- a/t4_devkit/sanity/reference/ref004.py +++ b/t4_devkit/sanity/reference/ref004.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF004(RecordReferenceChecker): id = RuleID("REF004") name = RuleName("sample-to-scene") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample.scene_token' refers to 'Scene' record." source = SchemaName.SAMPLE target = SchemaName.SCENE diff --git a/t4_devkit/sanity/reference/ref005.py b/t4_devkit/sanity/reference/ref005.py index 13ca061e..358d77c7 100644 --- a/t4_devkit/sanity/reference/ref005.py +++ b/t4_devkit/sanity/reference/ref005.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -18,6 +18,7 @@ class REF005(RecordReferenceChecker): id = RuleID("REF005") name = RuleName("sample-data-to-sample") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.sample_token' refers to 'Sample' record." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref006.py b/t4_devkit/sanity/reference/ref006.py index 01dbf62d..d44b926e 100644 --- a/t4_devkit/sanity/reference/ref006.py +++ b/t4_devkit/sanity/reference/ref006.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF006(RecordReferenceChecker): id = RuleID("REF006") name = RuleName("sample-data-to-ego-pose") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.ego_pose_token' refers to 'EgoPose' record." source = SchemaName.SAMPLE_DATA target = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/reference/ref009.py b/t4_devkit/sanity/reference/ref009.py index 61257e55..2ce220e2 100644 --- a/t4_devkit/sanity/reference/ref009.py +++ b/t4_devkit/sanity/reference/ref009.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF009(RecordReferenceChecker): id = RuleID("REF009") name = RuleName("instance-to-category") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Instance.category_token' refers to 'Category' record." source = SchemaName.INSTANCE target = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/reference/ref010.py b/t4_devkit/sanity/reference/ref010.py index daf38526..01eed0a9 100644 --- a/t4_devkit/sanity/reference/ref010.py +++ b/t4_devkit/sanity/reference/ref010.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF010(RecordReferenceChecker): id = RuleID("REF010") name = RuleName("instance-to-first-sample-annotation") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = ( "'Instance.first_annotation_token' refers to 'SampleAnnotation' or 'ObjectAnn' record." ) diff --git a/t4_devkit/sanity/reference/ref011.py b/t4_devkit/sanity/reference/ref011.py index 36f04956..f1a83d8a 100644 --- a/t4_devkit/sanity/reference/ref011.py +++ b/t4_devkit/sanity/reference/ref011.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF011(RecordReferenceChecker): id = RuleID("REF011") name = RuleName("instance-to-last-sample-annotation") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = ( "'Instance.last_annotation_token' refers to 'SampleAnnotation' or 'ObjectAnn' record." ) diff --git a/t4_devkit/sanity/reference/ref015.py b/t4_devkit/sanity/reference/ref015.py new file mode 100644 index 00000000..aaf080a4 --- /dev/null +++ b/t4_devkit/sanity/reference/ref015.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF015"] + + +@CHECKERS.register() +class REF015(RecordsChecker): + """A checker of REF015.""" + + id = RuleID("REF015") + name = RuleName("sample-data-to-sensor") + severity = Severity.ERROR + description = "'SampleData.sensor_token' refers to 'Sensor' record." + tables = ( + SchemaName.SAMPLE_DATA, + SchemaName.SENSOR, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references(context, SchemaName.SAMPLE_DATA, {"sensor_token": SchemaName.SENSOR}) + or None + ) diff --git a/t4_devkit/sanity/reference/ref016.py b/t4_devkit/sanity/reference/ref016.py new file mode 100644 index 00000000..63081728 --- /dev/null +++ b/t4_devkit/sanity/reference/ref016.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF016"] + + +@CHECKERS.register() +class REF016(RecordsChecker): + """A checker of REF016.""" + + id = RuleID("REF016") + name = RuleName("log-to-vehicle") + severity = Severity.ERROR + description = "'Log.vehicle_token' refers to 'Vehicle' record." + tables = ( + SchemaName.LOG, + SchemaName.VEHICLE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references(context, SchemaName.LOG, {"vehicle_token": SchemaName.VEHICLE}) + or None + ) diff --git a/t4_devkit/sanity/reference/ref017.py b/t4_devkit/sanity/reference/ref017.py new file mode 100644 index 00000000..8e09d13f --- /dev/null +++ b/t4_devkit/sanity/reference/ref017.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF017"] + + +@CHECKERS.register() +class REF017(RecordsChecker): + """A checker of REF017.""" + + id = RuleID("REF017") + name = RuleName("log-and-map") + severity = Severity.ERROR + description = ( + "'Log.map_token' refers to 'Map' record unless it is empty, 'Map.log_tokens' refer to " + "'Log' records, and both agree." + ) + tables = ( + SchemaName.LOG, + SchemaName.MAP, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = missing_references(context, SchemaName.LOG, {"map_token": SchemaName.MAP}) + failures += missing_references(context, SchemaName.MAP, {"log_tokens": SchemaName.LOG}) + maps = {map_.token: map_ for map_ in context.load_records(SchemaName.MAP).unwrap()} + for log in context.load_records(SchemaName.LOG).unwrap(): + if log.map_token in maps and log.token not in maps[log.map_token].log_tokens: + failures.append(Reason(f"Log {log.token}: its map does not list it")) + for map_ in maps.values(): + if log.token in map_.log_tokens and log.map_token != map_.token: + failures.append(Reason(f"Log {log.token}: listed by map {map_.token}")) + return failures or None diff --git a/t4_devkit/sanity/reference/ref018.py b/t4_devkit/sanity/reference/ref018.py new file mode 100644 index 00000000..2e49d35c --- /dev/null +++ b/t4_devkit/sanity/reference/ref018.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF018"] + + +@CHECKERS.register() +class REF018(RecordsChecker): + """A checker of REF018.""" + + id = RuleID("REF018") + name = RuleName("sample-annotation-references") + severity = Severity.ERROR + description = ( + "'SampleAnnotation' 'sample_token', 'instance_token', 'attribute_tokens' and " + "'visibility_token' (unless empty) refer to records." + ) + tables = ( + SchemaName.ATTRIBUTE, + SchemaName.INSTANCE, + SchemaName.SAMPLE, + SchemaName.SAMPLE_ANNOTATION, + SchemaName.VISIBILITY, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.SAMPLE_ANNOTATION, + { + "sample_token": SchemaName.SAMPLE, + "instance_token": SchemaName.INSTANCE, + "attribute_tokens": SchemaName.ATTRIBUTE, + "visibility_token": SchemaName.VISIBILITY, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref019.py b/t4_devkit/sanity/reference/ref019.py new file mode 100644 index 00000000..4d0c095b --- /dev/null +++ b/t4_devkit/sanity/reference/ref019.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF019"] + + +@CHECKERS.register() +class REF019(RecordsChecker): + """A checker of REF019.""" + + id = RuleID("REF019") + name = RuleName("object-ann-references") + severity = Severity.ERROR + description = ( + "'ObjectAnn' 'sample_data_token', 'instance_token' (unless empty), 'category_token' and " + "'attribute_tokens' refer to records." + ) + tables = ( + SchemaName.ATTRIBUTE, + SchemaName.CATEGORY, + SchemaName.INSTANCE, + SchemaName.OBJECT_ANN, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.OBJECT_ANN, + { + "sample_data_token": SchemaName.SAMPLE_DATA, + "instance_token": SchemaName.INSTANCE, + "category_token": SchemaName.CATEGORY, + "attribute_tokens": SchemaName.ATTRIBUTE, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref020.py b/t4_devkit/sanity/reference/ref020.py new file mode 100644 index 00000000..0f241cd7 --- /dev/null +++ b/t4_devkit/sanity/reference/ref020.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF020"] + + +@CHECKERS.register() +class REF020(RecordsChecker): + """A checker of REF020.""" + + id = RuleID("REF020") + name = RuleName("keypoint-references") + severity = Severity.ERROR + description = ( + "'Keypoint' 'sample_data_token', 'instance_token' and 'category_tokens' refer to records." + ) + tables = ( + SchemaName.CATEGORY, + SchemaName.INSTANCE, + SchemaName.KEYPOINT, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.KEYPOINT, + { + "sample_data_token": SchemaName.SAMPLE_DATA, + "instance_token": SchemaName.INSTANCE, + "category_tokens": SchemaName.CATEGORY, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref021.py b/t4_devkit/sanity/reference/ref021.py new file mode 100644 index 00000000..41ce35a4 --- /dev/null +++ b/t4_devkit/sanity/reference/ref021.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF021"] + + +@CHECKERS.register() +class REF021(RecordsChecker): + """A checker of REF021.""" + + id = RuleID("REF021") + name = RuleName("traffic-light-references") + severity = Severity.ERROR + description = ( + "'TrafficLight' 'object_ann_token' and 'instance_token' refer to records unless they are " + "empty." + ) + tables = ( + SchemaName.INSTANCE, + SchemaName.OBJECT_ANN, + SchemaName.TRAFFIC_LIGHT, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.TRAFFIC_LIGHT, + {"object_ann_token": SchemaName.OBJECT_ANN, "instance_token": SchemaName.INSTANCE}, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref022.py b/t4_devkit/sanity/reference/ref022.py new file mode 100644 index 00000000..b0609428 --- /dev/null +++ b/t4_devkit/sanity/reference/ref022.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF022"] + + +@CHECKERS.register() +class REF022(RecordsChecker): + """A checker of REF022.""" + + id = RuleID("REF022") + name = RuleName("route-references") + severity = Severity.ERROR + description = "'Route' 'log_token' and 'map_token' refer to records." + tables = ( + SchemaName.LOG, + SchemaName.MAP, + SchemaName.ROUTE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.ROUTE, + {"log_token": SchemaName.LOG, "map_token": SchemaName.MAP}, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref023.py b/t4_devkit/sanity/reference/ref023.py new file mode 100644 index 00000000..bc19af6c --- /dev/null +++ b/t4_devkit/sanity/reference/ref023.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF023"] + + +@CHECKERS.register() +class REF023(RecordsChecker): + """A checker of REF023.""" + + id = RuleID("REF023") + name = RuleName("scenario-references") + severity = Severity.ERROR + description = "'Scenario.log_token' refers to 'Log' record." + tables = ( + SchemaName.LOG, + SchemaName.SCENARIO, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references(context, SchemaName.SCENARIO, {"log_token": SchemaName.LOG}) or None + ) diff --git a/t4_devkit/sanity/reference/ref024.py b/t4_devkit/sanity/reference/ref024.py new file mode 100644 index 00000000..b4acddec --- /dev/null +++ b/t4_devkit/sanity/reference/ref024.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .base import missing_references + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF024"] + + +@CHECKERS.register() +class REF024(RecordsChecker): + """A checker of REF024.""" + + id = RuleID("REF024") + name = RuleName("caption-references") + severity = Severity.ERROR + description = ( + "'Caption' 'log_token', 'sample_data_token', 'scenario_token' and 'instance_tokens' refer " + "to records unless they are empty." + ) + tables = ( + SchemaName.CAPTION, + SchemaName.INSTANCE, + SchemaName.LOG, + SchemaName.SAMPLE_DATA, + SchemaName.SCENARIO, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + return ( + missing_references( + context, + SchemaName.CAPTION, + { + "log_token": SchemaName.LOG, + "sample_data_token": SchemaName.SAMPLE_DATA, + "scenario_token": SchemaName.SCENARIO, + "instance_tokens": SchemaName.INSTANCE, + }, + ) + or None + ) diff --git a/t4_devkit/sanity/reference/ref025.py b/t4_devkit/sanity/reference/ref025.py new file mode 100644 index 00000000..50f2dbfb --- /dev/null +++ b/t4_devkit/sanity/reference/ref025.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema import SensorModality +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF025"] + + +@CHECKERS.register() +class REF025(RecordsChecker): + """A checker of REF025.""" + + id = RuleID("REF025") + name = RuleName("lidar-source-channels") + severity = Severity.ERROR + description = "'Lidar.source_channels' name LiDAR 'Sensor' records that have no 'SampleData'." + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = context.load_records(SchemaName.SENSOR).unwrap() + with_frames = { + frame.sensor_token for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + } + sources = { + sensor.channel + for sensor in sensors + if sensor.modality == SensorModality.LIDAR and sensor.token not in with_frames + } + return [ + Reason( + f"Sensor {sensor.channel}: source channel {channel} is not a LiDAR without frames" + ) + for sensor in sensors + if sensor.lidar is not None + for channel in sensor.lidar.source_channels + if channel not in sources + ] or None diff --git a/t4_devkit/sanity/reference/ref101.py b/t4_devkit/sanity/reference/ref101.py index 4344f3c2..efe6bcc1 100644 --- a/t4_devkit/sanity/reference/ref101.py +++ b/t4_devkit/sanity/reference/ref101.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF101(RecordReferenceChecker): id = RuleID("REF101") name = RuleName("sample-next-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample.next' refers to another one unless it is empty." source = SchemaName.SAMPLE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref102.py b/t4_devkit/sanity/reference/ref102.py index abfaed49..7d90fb3e 100644 --- a/t4_devkit/sanity/reference/ref102.py +++ b/t4_devkit/sanity/reference/ref102.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF102(RecordReferenceChecker): id = RuleID("REF102") name = RuleName("sample-prev-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'Sample.prev' refers to another one unless it is empty." source = SchemaName.SAMPLE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref103.py b/t4_devkit/sanity/reference/ref103.py index ee956b79..c59fc5df 100644 --- a/t4_devkit/sanity/reference/ref103.py +++ b/t4_devkit/sanity/reference/ref103.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF103(RecordReferenceChecker): id = RuleID("REF103") name = RuleName("sample-annotation-next-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleAnnotation.next' refers to another one unless it is empty." source = SchemaName.SAMPLE_ANNOTATION target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref104.py b/t4_devkit/sanity/reference/ref104.py index ea363587..f980e06e 100644 --- a/t4_devkit/sanity/reference/ref104.py +++ b/t4_devkit/sanity/reference/ref104.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF104(RecordReferenceChecker): id = RuleID("REF104") name = RuleName("sample-annotation-prev-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleAnnotation.prev' refers to another one unless it is empty." source = SchemaName.SAMPLE_ANNOTATION target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref105.py b/t4_devkit/sanity/reference/ref105.py index 172dc5ce..cd9f0034 100644 --- a/t4_devkit/sanity/reference/ref105.py +++ b/t4_devkit/sanity/reference/ref105.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF105(RecordReferenceChecker): id = RuleID("REF105") name = RuleName("sample-data-next-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.next' refers to another one unless it is empty." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref106.py b/t4_devkit/sanity/reference/ref106.py index c6f0c454..a543b720 100644 --- a/t4_devkit/sanity/reference/ref106.py +++ b/t4_devkit/sanity/reference/ref106.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -16,6 +16,7 @@ class REF106(RecordReferenceChecker): id = RuleID("REF106") name = RuleName("sample-data-prev-to-another") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.prev' refers to another one unless it is empty." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref107.py b/t4_devkit/sanity/reference/ref107.py new file mode 100644 index 00000000..84de0aa1 --- /dev/null +++ b/t4_devkit/sanity/reference/ref107.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity, record_name +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF107"] + + +@CHECKERS.register() +class REF107(RecordsChecker): + """A checker of REF107.""" + + id = RuleID("REF107") + name = RuleName("links-mutual") + severity = Severity.ERROR + description = ( + "'next' and 'prev' agree: 'A.next' has 'prev' = 'A' ('Sample', 'SampleData', " + "'SampleAnnotation')." + ) + tables = ( + SchemaName.SAMPLE, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + failures = [] + for table in self.tables: + records = {record.token: record for record in context.load_records(table).unwrap()} + for record in records.values(): + following = records.get(record.next) + if following is not None and following.prev != record.token: + failures.append( + Reason(f"{record_name(table)} {record.token}: 'next' has another 'prev'") + ) + previous = records.get(record.prev) + if previous is not None and previous.next != record.token: + failures.append( + Reason(f"{record_name(table)} {record.token}: 'prev' has another 'next'") + ) + return failures or None diff --git a/t4_devkit/sanity/reference/ref108.py b/t4_devkit/sanity/reference/ref108.py new file mode 100644 index 00000000..6ae55d75 --- /dev/null +++ b/t4_devkit/sanity/reference/ref108.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF108"] + + +@CHECKERS.register() +class REF108(RecordsChecker): + """A checker of REF108.""" + + id = RuleID("REF108") + name = RuleName("links-same-stream") + severity = Severity.ERROR + description = ( + "'SampleData.next' is of the same channel, and 'SampleAnnotation.next' of the same " + "instance." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + SchemaName.SAMPLE_ANNOTATION, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + channels = { + sensor.token: sensor.channel + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + } + frames = { + frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + } + anns = { + ann.token: ann for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + } + failures = [ + Reason(f"SampleData {frame.token}: 'next' is of another channel") + for frame in frames.values() + if frame.next in frames + and channels.get(frames[frame.next].sensor_token) != channels.get(frame.sensor_token) + ] + failures += [ + Reason(f"SampleAnnotation {ann.token}: 'next' is of another instance") + for ann in anns.values() + if ann.next in anns and anns[ann.next].instance_token != ann.instance_token + ] + return failures or None diff --git a/t4_devkit/sanity/reference/ref201.py b/t4_devkit/sanity/reference/ref201.py index 168ebd86..e0fdf35d 100644 --- a/t4_devkit/sanity/reference/ref201.py +++ b/t4_devkit/sanity/reference/ref201.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName, Severity +from ..checker import RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -24,6 +24,7 @@ class REF201(FileReferenceChecker): id = RuleID("REF201") name = RuleName("sample-data-filename-presence") severity = Severity.ERROR + versions = (SchemaVersion.V0, SchemaVersion.V1) description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA @@ -34,5 +35,7 @@ def check(self, context: SanityContext) -> list[Reason] | None: return [ Reason(f"File not found: {record['filename']}") for record in records - if record.get("is_valid", True) and not data_root.joinpath(record["filename"]).exists() + if record.get("is_valid", True) + and record["filename"] is not None # 1.0: no file + and not data_root.joinpath(record["filename"]).exists() ] or None diff --git a/t4_devkit/sanity/reference/ref204.py b/t4_devkit/sanity/reference/ref204.py new file mode 100644 index 00000000..1f50e8cd --- /dev/null +++ b/t4_devkit/sanity/reference/ref204.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from ..record.base import channel_file + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF204"] + + +@CHECKERS.register() +class REF204(RecordsChecker): + """A checker of REF204.""" + + id = RuleID("REF204") + name = RuleName("sample-data-file-of-sensor") + severity = Severity.ERROR + description = ( + "For a channel stored in one file, 'SampleData.filename' is its sensor's " + "'camera.video.path' / 'lidar.container.path'." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + files = { + sensor.token: stored[0] + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + if (stored := channel_file(sensor)) is not None + } + return [ + Reason( + f"SampleData {frame.token}: 'filename' {frame.filename}, sensor file {files[frame.sensor_token]}" + ) + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap() + if frame.sensor_token in files and frame.filename != files[frame.sensor_token] + ] or None diff --git a/t4_devkit/sanity/reference/ref205.py b/t4_devkit/sanity/reference/ref205.py new file mode 100644 index 00000000..91a00425 --- /dev/null +++ b/t4_devkit/sanity/reference/ref205.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .files import describe_lidar_file, describe_video_file + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF205"] + + +@CHECKERS.register() +class REF205(RecordsChecker): + """A checker of REF205.""" + + id = RuleID("REF205") + name = RuleName("sensor-file-description") + severity = Severity.ERROR + description = ( + "Each channel file matches its 'sensor' row: Parquet columns, dtypes and one row group per " + "frame; video codec, pixel format, bit depth, timescale, frame count and frame size." + ) + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + failures = [] + for sensor in context.load_records(SchemaName.SENSOR).unwrap(): + if sensor.camera is not None and sensor.camera.video is not None: + path = data_root.joinpath(sensor.camera.video.path) + if path.exists(): + failures += describe_video_file(sensor, path) + elif sensor.lidar is not None and sensor.lidar.container.path is not None: + path = data_root.joinpath(sensor.lidar.container.path) + if path.exists() and sensor.lidar.container.format.value == "parquet": + failures += describe_lidar_file(sensor, path) + return failures or None diff --git a/t4_devkit/sanity/reference/ref206.py b/t4_devkit/sanity/reference/ref206.py new file mode 100644 index 00000000..bfac4166 --- /dev/null +++ b/t4_devkit/sanity/reference/ref206.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pyarrow.parquet as pq + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from pathlib import Path + + from ..context import SanityContext + + +__all__ = ["REF206"] + + +@CHECKERS.register() +class REF206(RecordsChecker): + """A checker of REF206.""" + + id = RuleID("REF206") + name = RuleName("lidar-num-points") + severity = Severity.ERROR + description = "'SampleData.num_points' equals the number of points in its frame, when set." + tables = ( + SchemaName.SENSOR, + SchemaName.SAMPLE_DATA, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + files = { + sensor.token: data_root.joinpath(sensor.lidar.container.path) + for sensor in context.load_records(SchemaName.SENSOR).unwrap() + if sensor.lidar is not None + and sensor.lidar.container.path is not None + and sensor.lidar.container.format.value == "parquet" + } + row_counts = { + token: _row_group_sizes(path) for token, path in files.items() if path.exists() + } + failures = [] + for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): + counts = row_counts.get(frame.sensor_token) + if counts is None or frame.num_points is None or frame.frame_index is None: + continue + if frame.frame_index >= len(counts) or counts[frame.frame_index] != frame.num_points: + failures.append( + Reason( + f"SampleData {frame.token}: 'num_points' {frame.num_points} differs from its frame" + ) + ) + return failures or None + + +def _row_group_sizes(path: Path) -> list[int]: + metadata = pq.ParquetFile(path).metadata + return [metadata.row_group(i).num_rows for i in range(metadata.num_row_groups)] diff --git a/t4_devkit/sanity/reference/ref207.py b/t4_devkit/sanity/reference/ref207.py new file mode 100644 index 00000000..ccf04fdc --- /dev/null +++ b/t4_devkit/sanity/reference/ref207.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import hashlib + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF207"] + + +@CHECKERS.register() +class REF207(RecordsChecker): + """A checker of REF207.""" + + id = RuleID("REF207") + name = RuleName("map-file") + severity = Severity.ERROR + description = "'Map.filename' exists and has the SHA-256 'Map.sha256'." + tables = (SchemaName.MAP,) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + failures = [] + for map_ in context.load_records(SchemaName.MAP).unwrap(): + path = data_root.joinpath(map_.filename) + if not path.exists(): + failures.append(Reason(f"Map {map_.token}: '{map_.filename}' not found")) + elif hashlib.sha256(path.read_bytes()).hexdigest() != map_.sha256: + failures.append(Reason(f"Map {map_.token}: SHA-256 of '{map_.filename}' differs")) + return failures or None diff --git a/t4_devkit/sanity/reference/ref302.py b/t4_devkit/sanity/reference/ref302.py new file mode 100644 index 00000000..aee269ae --- /dev/null +++ b/t4_devkit/sanity/reference/ref302.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF302"] + + +@CHECKERS.register() +class REF302(RecordsChecker): + """A checker of REF302.""" + + id = RuleID("REF302") + name = RuleName("instance-to-map-element") + severity = Severity.ERROR + description = "'Instance.map_element' refers to an element of its 'type' in the lanelet2 map." + tables = ( + SchemaName.MAP, + SchemaName.INSTANCE, + ) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + skip = super().can_skip(context) + if skip != Nothing: + return skip + if context.lanelet_map == Nothing: + return Maybe.from_value(Reason("No lanelet2 map to check against")) + return Nothing + + def check(self, context: SanityContext) -> list[Reason] | None: + lanelet = context.lanelet_map.unwrap() + elements = { + "node": set(lanelet.nodes), + "way": set(lanelet.ways), + "relation": set(lanelet.relations), + } + return [ + Reason(f"Instance {instance.token}: no {element.type.value} {element.id} in the map") + for instance in context.load_records(SchemaName.INSTANCE).unwrap() + if (element := instance.map_element).id is not None + and str(element.id) not in elements[element.type.value] + ] or None diff --git a/t4_devkit/sanity/reference/ref303.py b/t4_devkit/sanity/reference/ref303.py new file mode 100644 index 00000000..936ead3f --- /dev/null +++ b/t4_devkit/sanity/reference/ref303.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF303"] + + +@CHECKERS.register() +class REF303(RecordsChecker): + """A checker of REF303.""" + + id = RuleID("REF303") + name = RuleName("traffic-light-to-lanelet") + severity = Severity.ERROR + description = ( + "'TrafficLight.regulatory_element_id' is a traffic-light regulatory element of the " + "lanelet2 map." + ) + tables = ( + SchemaName.MAP, + SchemaName.TRAFFIC_LIGHT, + ) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + skip = super().can_skip(context) + if skip != Nothing: + return skip + if context.lanelet_map == Nothing: + return Maybe.from_value(Reason("No lanelet2 map to check against")) + return Nothing + + def check(self, context: SanityContext) -> list[Reason] | None: + regulatory_elements = { + id + for id, relation in context.lanelet_map.unwrap().relations.items() + if relation.tags.get("type") == "regulatory_element" + and relation.tags.get("subtype") == "traffic_light" + } + return [ + Reason( + f"TrafficLight {light.token}: no traffic-light regulatory element {light.regulatory_element_id}" + ) + for light in context.load_records(SchemaName.TRAFFIC_LIGHT).unwrap() + if str(light.regulatory_element_id) not in regulatory_elements + ] or None diff --git a/t4_devkit/sanity/reference/ref304.py b/t4_devkit/sanity/reference/ref304.py new file mode 100644 index 00000000..a8e861ce --- /dev/null +++ b/t4_devkit/sanity/reference/ref304.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Maybe, Nothing + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF304"] + + +@CHECKERS.register() +class REF304(RecordsChecker): + """A checker of REF304.""" + + id = RuleID("REF304") + name = RuleName("route-to-lanelet") + severity = Severity.ERROR + description = "'Route' segment ids are lanelets of the lanelet2 map." + tables = ( + SchemaName.MAP, + SchemaName.ROUTE, + ) + + def can_skip(self, context: SanityContext) -> Maybe[Reason]: + skip = super().can_skip(context) + if skip != Nothing: + return skip + if context.lanelet_map == Nothing: + return Maybe.from_value(Reason("No lanelet2 map to check against")) + return Nothing + + def check(self, context: SanityContext) -> list[Reason] | None: + lanelets = { + id + for id, relation in context.lanelet_map.unwrap().relations.items() + if relation.tags.get("type") == "lanelet" + } + return [ + Reason(f"Route {route.token}: no lanelet {id}") + for route in context.load_records(SchemaName.ROUTE).unwrap() + for segment in route.segments + for id in sorted({segment.preferred_primitive_id, *segment.primitive_ids}) + if str(id) not in lanelets + ] or None diff --git a/t4_devkit/sanity/reference/ref305.py b/t4_devkit/sanity/reference/ref305.py new file mode 100644 index 00000000..c2c6bbad --- /dev/null +++ b/t4_devkit/sanity/reference/ref305.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .files import lidar_column_values + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF305"] + + +@CHECKERS.register() +class REF305(RecordsChecker): + """A checker of REF305.""" + + id = RuleID("REF305") + name = RuleName("lidar-source-index") + severity = Severity.ERROR + description = ( + "In a merged LiDAR file, 'source_index' values are the 'Sensor.index' of the channel's " + "'source_channels'." + ) + tables = (SchemaName.SENSOR,) + + def check(self, context: SanityContext) -> list[Reason] | None: + sensors = context.load_records(SchemaName.SENSOR).unwrap() + data_root = context.data_root.unwrap() + failures = [] + for sensor in sensors: + if sensor.lidar is None or not sensor.lidar.source_channels: + continue + path = sensor.lidar.container.path + if path is None or not data_root.joinpath(path).exists(): + continue + expected = {s.index for s in sensors if s.channel in sensor.lidar.source_channels} + values = lidar_column_values(data_root.joinpath(path), "source_index") + if values is None: + failures.append(Reason(f"Sensor {sensor.channel}: no 'source_index' column")) + elif not values <= expected: + failures.append( + Reason( + f"Sensor {sensor.channel}: 'source_index' values {sorted(values - expected)} are not its sources" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/reference/ref306.py b/t4_devkit/sanity/reference/ref306.py new file mode 100644 index 00000000..944df3fc --- /dev/null +++ b/t4_devkit/sanity/reference/ref306.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import RecordsChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason +from .files import lidar_column_values + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["REF306"] + + +@CHECKERS.register() +class REF306(RecordsChecker): + """A checker of REF306.""" + + id = RuleID("REF306") + name = RuleName("lidar-point-labels") + severity = Severity.ERROR + description = ( + "In a LiDAR file, 'label' values are 'Category.index' values, and 'instance' values are " + "'Instance.index' values or -1 (no object)." + ) + tables = ( + SchemaName.SENSOR, + SchemaName.CATEGORY, + SchemaName.INSTANCE, + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + indices = { + "label": {c.index for c in context.load_records(SchemaName.CATEGORY).unwrap()}, + "instance": {-1} + | {i.index for i in context.load_records(SchemaName.INSTANCE).unwrap()}, + } + failures = [] + for sensor in context.load_records(SchemaName.SENSOR).unwrap(): + if sensor.lidar is None or sensor.lidar.container.path is None: + continue + path = data_root.joinpath(sensor.lidar.container.path) + if not path.exists(): + continue + for column, known in indices.items(): + values = lidar_column_values(path, column) + if values is not None and not values <= known: + failures.append( + Reason( + f"Sensor {sensor.channel}: '{column}' values {sorted(values - known)} not defined" + ) + ) + return failures or None diff --git a/t4_devkit/sanity/registry.py b/t4_devkit/sanity/registry.py index 322dbb0c..f3459307 100644 --- a/t4_devkit/sanity/registry.py +++ b/t4_devkit/sanity/registry.py @@ -4,7 +4,7 @@ from collections.abc import Callable, Sequence from enum import Enum, unique -from .checker import Checker, RuleID +from .checker import Checker, RuleID, SchemaVersion @unique @@ -69,11 +69,17 @@ def _add_module(self, module: type[Checker]) -> None: self[group][module.id] = module - def build(self, excludes: Sequence[str] | None = None) -> list[Checker]: + def build( + self, + excludes: Sequence[str] | None = None, + version: SchemaVersion | None = None, + ) -> list[Checker]: """Build a list of checkers from the registry. Args: excludes (Sequence[str] | None, optional): A list of rule IDs or rule groups to exclude. + version (SchemaVersion | None, optional): Keep only the rules of this schema version; + all rules if None. Returns: A list of checkers. @@ -85,7 +91,9 @@ def build(self, excludes: Sequence[str] | None = None) -> list[Checker]: checker() for group, values in self.items() for id, checker in values.items() - if id not in excludes and group.value not in excludes + if id not in excludes + and group.value not in excludes + and (version is None or version in checker.versions) ] diff --git a/t4_devkit/sanity/run.py b/t4_devkit/sanity/run.py index 30f0663b..3127caf9 100644 --- a/t4_devkit/sanity/run.py +++ b/t4_devkit/sanity/run.py @@ -20,6 +20,7 @@ def sanity_check( *, excludes: Sequence[str] | None = None, fix: bool = False, + checksum: bool = False, ) -> SanityResult: """Run sanity checks on the given data root. @@ -28,13 +29,16 @@ def sanity_check( revision (str | None, optional): The revision to check. If None, the latest revision is used. excludes (Sequence[str] | None, optional): A list of rule names or groups to exclude. fix (bool, optional): Attempt to fix the issues reported by the sanity check. + checksum (bool, optional): Verify the SHA-256 of every file listed in `manifest.json` + (1.0 only). Reads every file, including videos and rosbags. Returns: A SanityResult object. """ - context = SanityContext.from_path(data_root, revision=revision) + context = SanityContext.from_path(data_root, revision=revision, checksum=checksum) - checkers = CHECKERS.build(excludes=excludes) + # only the rules of the dataset's schema version + checkers = CHECKERS.build(excludes=excludes, version=context.schema_version) reports = [checker(context, fix=fix) for checker in checkers] return SanityResult.from_context(context, reports) diff --git a/t4_devkit/sanity/safety.py b/t4_devkit/sanity/safety.py index 8ec12807..b2f2b66c 100644 --- a/t4_devkit/sanity/safety.py +++ b/t4_devkit/sanity/safety.py @@ -6,6 +6,7 @@ from t4_devkit import DBMetadata, T4Devkit, load_metadata from t4_devkit.common import load_json, save_json +from t4_devkit.schema.v1 import Manifest if TYPE_CHECKING: from t4_devkit.schema import SchemaBase @@ -32,6 +33,12 @@ def load_schema_safe(module: type[SchemaBase], record: dict) -> SchemaBase: return module.from_dict(record) +@safe +def load_manifest_safe(data: dict) -> Manifest: + """Build the 1.0 manifest from its dict safely.""" + return Manifest(**data) + + @safe def load_metadata_safe(data_root: PathLike, revision: str | None = None) -> DBMetadata: """Load DBMetadata safely.""" diff --git a/t4_devkit/sanity/structure/__init__.py b/t4_devkit/sanity/structure/__init__.py index 0d1224d7..9430b102 100644 --- a/t4_devkit/sanity/structure/__init__.py +++ b/t4_devkit/sanity/structure/__init__.py @@ -9,3 +9,9 @@ from .str007 import * # noqa from .str008 import * # noqa from .str009 import * # noqa +from .str010 import * # noqa +from .str011 import * # noqa +from .str012 import * # noqa +from .str013 import * # noqa +from .str014 import * # noqa +from .str015 import * # noqa diff --git a/t4_devkit/sanity/structure/str001.py b/t4_devkit/sanity/structure/str001.py index 9894e6d1..8316ff77 100644 --- a/t4_devkit/sanity/structure/str001.py +++ b/t4_devkit/sanity/structure/str001.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR001(Checker): name = RuleName("version-dir-presence") severity = Severity.WARNING description = "'version/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.version: diff --git a/t4_devkit/sanity/structure/str002.py b/t4_devkit/sanity/structure/str002.py index 3a081faf..c4729eb0 100644 --- a/t4_devkit/sanity/structure/str002.py +++ b/t4_devkit/sanity/structure/str002.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR002(Checker): name = RuleName("annotation-dir-presence") severity = Severity.ERROR description = "'annotation/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.annotation_dir: diff --git a/t4_devkit/sanity/structure/str003.py b/t4_devkit/sanity/structure/str003.py index e83b3f16..6be6d209 100644 --- a/t4_devkit/sanity/structure/str003.py +++ b/t4_devkit/sanity/structure/str003.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR003(Checker): name = RuleName("data-dir-presence") severity = Severity.ERROR description = "'data/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.sensor_data_dir: diff --git a/t4_devkit/sanity/structure/str004.py b/t4_devkit/sanity/structure/str004.py index 2d0101ef..2d6a274f 100644 --- a/t4_devkit/sanity/structure/str004.py +++ b/t4_devkit/sanity/structure/str004.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR004(Checker): name = RuleName("map-dir-presence") severity = Severity.WARNING description = "'map/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: diff --git a/t4_devkit/sanity/structure/str005.py b/t4_devkit/sanity/structure/str005.py index 911ca56c..702a96bf 100644 --- a/t4_devkit/sanity/structure/str005.py +++ b/t4_devkit/sanity/structure/str005.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -23,6 +23,7 @@ class STR005(Checker): name = RuleName("bag-dir-presence") severity = Severity.WARNING description = "'input_bag/' directory exists under the dataset root directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.bag_dir: diff --git a/t4_devkit/sanity/structure/str008.py b/t4_devkit/sanity/structure/str008.py index 95033016..0942bfc3 100644 --- a/t4_devkit/sanity/structure/str008.py +++ b/t4_devkit/sanity/structure/str008.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -22,6 +22,7 @@ class STR008(Checker): name = RuleName("lanelet-file-presence") severity = Severity.WARNING description = "'lanelet2_map.osm' file exists under the 'map/' directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: diff --git a/t4_devkit/sanity/structure/str009.py b/t4_devkit/sanity/structure/str009.py index 24ba4f40..77a8af43 100644 --- a/t4_devkit/sanity/structure/str009.py +++ b/t4_devkit/sanity/structure/str009.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason @@ -22,6 +22,7 @@ class STR009(Checker): name = RuleName("pointcloud-map-dir-presence") severity = Severity.WARNING description = "'pointcloud_map.pcd' directory exists under the 'map/' directory." + versions = (SchemaVersion.V0, SchemaVersion.V1) def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: diff --git a/t4_devkit/sanity/structure/str010.py b/t4_devkit/sanity/structure/str010.py new file mode 100644 index 00000000..17b29253 --- /dev/null +++ b/t4_devkit/sanity/structure/str010.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Some + +from t4_devkit.schema.v1 import MANIFEST_FILENAME, SUPPORTED_SCHEMA_MAJOR_VERSION + +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR010"] + + +@CHECKERS.register() +class STR010(Checker): + """A checker of STR010.""" + + id = RuleID("STR010") + name = RuleName("schema-version-supported") + severity = Severity.ERROR + description = ( + "'manifest.json' declares a supported 'schema_version' (major version 1). " + "If not, no other 1.0 rule runs." + ) + versions = (SchemaVersion.V1,) + + def check(self, context: SanityContext) -> list[Reason] | None: + match context.raw_manifest: + case Some(manifest): + version = manifest.get("schema_version") if isinstance(manifest, dict) else None + if not isinstance(version, str): + return [Reason(f"'schema_version' is missing or not a string: {version!r}")] + if version.split(".")[0] != str(SUPPORTED_SCHEMA_MAJOR_VERSION): + return [Reason(f"Unsupported 'schema_version': {version}")] + return None + return [Reason(f"'{MANIFEST_FILENAME}' cannot be read")] diff --git a/t4_devkit/sanity/structure/str011.py b/t4_devkit/sanity/structure/str011.py new file mode 100644 index 00000000..aed78f44 --- /dev/null +++ b/t4_devkit/sanity/structure/str011.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR011"] + + +@CHECKERS.register() +class STR011(ManifestChecker): + """A checker of STR011.""" + + id = RuleID("STR011") + name = RuleName("manifest-files-presence") + severity = Severity.ERROR + description = "Every file listed in 'manifest.json' ('tables' and 'assets') exists." + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + return [ + Reason(f"Listed file not found: {path}") + for path in context.manifest.unwrap().listed_paths() + if not data_root.joinpath(path).exists() + ] or None diff --git a/t4_devkit/sanity/structure/str012.py b/t4_devkit/sanity/structure/str012.py new file mode 100644 index 00000000..76da8761 --- /dev/null +++ b/t4_devkit/sanity/structure/str012.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import AssetEntry + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason, make_report + +if TYPE_CHECKING: + from pathlib import Path + + from ..context import SanityContext + from ..result import Report + + +__all__ = ["STR012"] + + +@CHECKERS.register() +class STR012(ManifestChecker): + """A checker of STR012. + + Opt-in (`checksum=True`, `t4sanity --checksum`), since it reads every file, including videos + and rosbags. Without it, the rule reports a warning that hashes were not verified. + """ + + id = RuleID("STR012") + name = RuleName("manifest-files-checksum") + severity = Severity.ERROR + description = ( + "Every listed file has the listed 'sha256', and every asset the listed 'bytes'. " + "Opt-in ('--checksum'); a warning if not run." + ) + + def __call__(self, context: SanityContext, fix: bool = False) -> Report: + if not context.checksum: + reasons = [Reason("Hashes not verified: run with '--checksum'")] + return make_report(self.id, self.name, Severity.WARNING, self.description, reasons) + return super().__call__(context, fix=fix) + + def check(self, context: SanityContext) -> list[Reason] | None: + manifest = context.manifest.unwrap() + data_root = context.data_root.unwrap() + failures = [] + entries = [*manifest.tables.values(), *manifest.assets] + for entry in entries: + filepath = data_root.joinpath(entry.path) + if not filepath.exists(): + continue # reported by STR011 + if isinstance(entry, AssetEntry) and filepath.stat().st_size != entry.bytes: + failures.append( + Reason(f"{entry.path}: {filepath.stat().st_size} bytes, listed {entry.bytes}") + ) + elif _sha256(filepath) != entry.sha256: + failures.append(Reason(f"{entry.path}: SHA-256 differs from 'manifest.json'")) + return failures or None + + +def _sha256(filepath: Path) -> str: + digest = hashlib.sha256() + with filepath.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/t4_devkit/sanity/structure/str013.py b/t4_devkit/sanity/structure/str013.py new file mode 100644 index 00000000..1db29f65 --- /dev/null +++ b/t4_devkit/sanity/structure/str013.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR013"] + +# Directories whose files must all be listed in `manifest.json`. +LISTED_DIRS = ("annotation", "data", "map", "input_bag") + + +@CHECKERS.register() +class STR013(ManifestChecker): + """A checker of STR013.""" + + id = RuleID("STR013") + name = RuleName("manifest-files-complete") + severity = Severity.WARNING + description = ( + "Every file under 'annotation/', 'data/', 'map/' and 'input_bag/' is listed in " + "'manifest.json'." + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + data_root = context.data_root.unwrap() + listed = set(context.manifest.unwrap().listed_paths()) + return [ + Reason(f"File not listed: {path}") + for directory in LISTED_DIRS + for path in sorted( + filepath.relative_to(data_root).as_posix() + for filepath in data_root.joinpath(directory).rglob("*") + if filepath.is_file() + ) + if path not in listed + ] or None diff --git a/t4_devkit/sanity/structure/str014.py b/t4_devkit/sanity/structure/str014.py new file mode 100644 index 00000000..96ed1dfb --- /dev/null +++ b/t4_devkit/sanity/structure/str014.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from t4_devkit.schema.v1 import SchemaName + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR014"] + + +@CHECKERS.register() +class STR014(ManifestChecker): + """A checker of STR014.""" + + id = RuleID("STR014") + name = RuleName("manifest-required-tables") + severity = Severity.ERROR + description = "'manifest.json' lists every required table." + + def check(self, context: SanityContext) -> list[Reason] | None: + tables = context.manifest.unwrap().tables + return [ + Reason(f"Required table not listed: {table.value}") + for table in SchemaName + if not table.is_optional() and table.value not in tables + ] or None diff --git a/t4_devkit/sanity/structure/str015.py b/t4_devkit/sanity/structure/str015.py new file mode 100644 index 00000000..6107777d --- /dev/null +++ b/t4_devkit/sanity/structure/str015.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from returns.maybe import Some + +from ..checker import ManifestChecker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + +if TYPE_CHECKING: + from ..context import SanityContext + + +__all__ = ["STR015"] + + +@CHECKERS.register() +class STR015(ManifestChecker): + """A checker of STR015.""" + + id = RuleID("STR015") + name = RuleName("manifest-identity-matches-dirs") + severity = Severity.WARNING + description = ( + "'manifest.json' 'dataset_id' and 'version' match the names of the dataset and version " + "directories." + ) + + def check(self, context: SanityContext) -> list[Reason] | None: + manifest = context.manifest.unwrap() + failures = [] + if manifest.dataset_id != context.dataset_id.unwrap(): + failures.append( + Reason( + f"'dataset_id' {manifest.dataset_id!r} differs from the directory " + f"{context.dataset_id.unwrap()!r}" + ) + ) + match context.version: + case Some(directory) if str(manifest.version) != directory: + failures.append( + Reason(f"'version' {manifest.version} differs from the directory {directory!r}") + ) + return failures or None diff --git a/t4_devkit/sanity/tier4/tiv001.py b/t4_devkit/sanity/tier4/tiv001.py index 25f46ebd..8a1a24c6 100644 --- a/t4_devkit/sanity/tier4/tiv001.py +++ b/t4_devkit/sanity/tier4/tiv001.py @@ -5,7 +5,7 @@ from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful -from ..checker import Checker, RuleID, RuleName, Severity +from ..checker import Checker, RuleID, RuleName, SchemaVersion, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_tier4_safe @@ -24,6 +24,7 @@ class TIV001(Checker): name = RuleName("load-tier4") severity = Severity.ERROR description = "Ensure 'T4Devkit' instance is loaded successfully." + versions = (SchemaVersion.V0, SchemaVersion.V1) def can_skip(self, context: SanityContext) -> Maybe[Reason]: match context.data_root: diff --git a/t4_devkit/schema/v1/manifest.py b/t4_devkit/schema/v1/manifest.py index ff5f9c3c..7a9a1667 100644 --- a/t4_devkit/schema/v1/manifest.py +++ b/t4_devkit/schema/v1/manifest.py @@ -295,6 +295,16 @@ class Manifest: ) ) + def listed_paths(self) -> list[str]: + """Return the path of every file listed: tables, then assets. + + Returns: + Paths relative to the dataset root. + """ + return [entry.path for entry in self.tables.values()] + [ + entry.path for entry in self.assets + ] + @schema_version.validator def _check_schema_version(self, attribute, value: str) -> None: major = value.split(".")[0] diff --git a/tests/sample/t4dataset_v1/annotation/map.json b/tests/sample/t4dataset_v1/annotation/map.json index d712d886..5cdd7134 100644 --- a/tests/sample/t4dataset_v1/annotation/map.json +++ b/tests/sample/t4dataset_v1/annotation/map.json @@ -5,7 +5,7 @@ "log_tokens": ["log"], "format": "lanelet2_osm", "filename": "map/lanelet2_map.osm", - "sha256": "fcb070e447b1d6b529b2b641b9a42bb2f628447a76e571566ec4b5beeba423c2", + "sha256": "4b325cf23adf604023824cc4a3243a7aac56c690fa872b68ce58e42a4529f054", "map_id": "sample_map", "version_id": "sample_map_version", "projector": { diff --git a/tests/sample/t4dataset_v1/annotation/route.json b/tests/sample/t4dataset_v1/annotation/route.json index f9a6ea4e..924c9de0 100644 --- a/tests/sample/t4dataset_v1/annotation/route.json +++ b/tests/sample/t4dataset_v1/annotation/route.json @@ -16,8 +16,8 @@ }, "segments": [ { - "preferred_primitive_id": 1, - "primitive_ids": [1] + "preferred_primitive_id": 1000, + "primitive_ids": [1000, 1001] } ], "allow_modification": false, diff --git a/tests/sample/t4dataset_v1/annotation/traffic_light.json b/tests/sample/t4dataset_v1/annotation/traffic_light.json index 10dc7f4e..eacec538 100644 --- a/tests/sample/t4dataset_v1/annotation/traffic_light.json +++ b/tests/sample/t4dataset_v1/annotation/traffic_light.json @@ -2,7 +2,7 @@ { "token": "traffic_light0", "timestamp": 1704067200000000, - "regulatory_element_id": 1, + "regulatory_element_id": 2000, "elements": [ { "color": "green", diff --git a/tests/sample/t4dataset_v1/make_sample_v1.py b/tests/sample/t4dataset_v1/make_sample_v1.py index ec7ccc36..b385d703 100644 --- a/tests/sample/t4dataset_v1/make_sample_v1.py +++ b/tests/sample/t4dataset_v1/make_sample_v1.py @@ -108,10 +108,34 @@ def write_lidar() -> None: writer.write_table(table, row_group_size=NUM_POINTS) +# Added to the 0.x sample's map: a traffic light (way) and its regulatory element, controlling +# the main road lanelet 1000. +TRAFFIC_LIGHT_WAY = 200 +TRAFFIC_LIGHT_REGULATORY_ELEMENT = 2000 +LANELETS = [1000, 1001] +MAP_ADDITIONS = f""" + + + + + + + + + + + + + + +""" + + def write_map() -> None: path = ROOT / MAP_PATH path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes((ROOT.parent / "t4dataset" / MAP_PATH).read_bytes()) + osm = (ROOT.parent / "t4dataset" / MAP_PATH).read_text() + path.write_text(osm.replace("\n\n", MAP_ADDITIONS)) def label( @@ -426,7 +450,7 @@ def link(tokens: list[str], i: int) -> dict: { "token": "traffic_light0", "timestamp": timestamps[0], - "regulatory_element_id": 1, + "regulatory_element_id": TRAFFIC_LIGHT_REGULATORY_ELEMENT, "elements": [ {"color": "green", "shape": "circle", "status": "solid_on", "confidence": 1.0} ], @@ -445,7 +469,7 @@ def link(tokens: list[str], i: int) -> dict: "route_uuid": None, "start_pose": {"translation": zero, "rotation": identity}, "goal_pose": {"translation": [100.0, 0.0, 0.0], "rotation": identity}, - "segments": [{"preferred_primitive_id": 1, "primitive_ids": [1]}], + "segments": [{"preferred_primitive_id": LANELETS[0], "primitive_ids": LANELETS}], "allow_modification": False, "source": "mission_planner", "provenance": "own_bag", diff --git a/tests/sample/t4dataset_v1/manifest.json b/tests/sample/t4dataset_v1/manifest.json index 669c53d7..109a399c 100644 --- a/tests/sample/t4dataset_v1/manifest.json +++ b/tests/sample/t4dataset_v1/manifest.json @@ -60,7 +60,7 @@ }, "map": { "path": "annotation/map.json", - "sha256": "a77271964d953ac5ddfce7aed69cb7d66937ac30017be38f341626ec0776e23c" + "sha256": "d869de9d95d4ed8560ab8463c9e9aac442e2d07f03f8ce4d798cdd51828bd3b8" }, "vehicle_state": { "path": "annotation/vehicle_state.json", @@ -96,11 +96,11 @@ }, "traffic_light": { "path": "annotation/traffic_light.json", - "sha256": "0d5284ca4ed203a941390215c6f2cd5f311c1ed04d1afb077dbbdbba0777b7c4" + "sha256": "989e24ff88fe14aaf6da6d524b6dbfea7e96fd36a67407cf8f16193674a9239d" }, "route": { "path": "annotation/route.json", - "sha256": "09f0fe9f01049a3f331146068332417af5130db6258602585536ead14ae091ca" + "sha256": "6badf9c9fc00722b62763bc483f24e04a53df9cc5a717130008581b41693cecb" }, "scenario": { "path": "annotation/scenario.json", @@ -124,8 +124,8 @@ }, { "path": "map/lanelet2_map.osm", - "sha256": "fcb070e447b1d6b529b2b641b9a42bb2f628447a76e571566ec4b5beeba423c2", - "bytes": 5661 + "sha256": "4b325cf23adf604023824cc4a3243a7aac56c690fa872b68ce58e42a4529f054", + "bytes": 6055 } ], "provenance": { diff --git a/tests/sample/t4dataset_v1/map/lanelet2_map.osm b/tests/sample/t4dataset_v1/map/lanelet2_map.osm index 6f7149a4..98ea03d3 100644 --- a/tests/sample/t4dataset_v1/map/lanelet2_map.osm +++ b/tests/sample/t4dataset_v1/map/lanelet2_map.osm @@ -192,4 +192,17 @@ + + + + + + + + + + + + + diff --git a/tests/sanity/test_v1_checkers.py b/tests/sanity/test_v1_checkers.py new file mode 100644 index 00000000..abdc4a8e --- /dev/null +++ b/tests/sanity/test_v1_checkers.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from t4_devkit.sanity import sanity_check +from t4_devkit.sanity.checker import SchemaVersion +from t4_devkit.sanity.registry import CHECKERS + +SAMPLE_ROOT = Path(__file__).parents[1] / "sample" +SAMPLE_V0 = SAMPLE_ROOT / "t4dataset" +SAMPLE_V1 = SAMPLE_ROOT / "t4dataset_v1" + + +def _reports(data_root: Path, **kwargs) -> dict: + return {report.id: report for report in sanity_check(data_root.as_posix(), **kwargs).reports} + + +def _failed(reports: dict) -> set[str]: + return {id for id, report in reports.items() if report.is_failed()} + + +def _copy_v1(tmp_path: Path) -> Path: + data_root = tmp_path / "sample" # named as the sample's `dataset_id` + shutil.copytree(SAMPLE_V1, data_root) + return data_root + + +def _edit_json(path: Path, change) -> None: + path.write_text(json.dumps(change(json.loads(path.read_text())))) + + +def test_rules_by_version() -> None: + """Each dataset gets only the rules of its version; 0.x keeps exactly its 0.x rules.""" + v0_rules = { + id + for rules in CHECKERS.values() + for id, checker in rules.items() + if SchemaVersion.V0 in checker.versions + } + assert set(_reports(SAMPLE_V0)) == v0_rules + assert len(v0_rules) == 60 + + v1_reports = _reports(SAMPLE_V1) + assert "STR006" not in v1_reports and "FMT001" not in v1_reports + assert {"STR001", "TIV001", "STR010", "FMT020", "FMT040"} <= set(v1_reports) + + +def test_v1_sample_passes(tmp_path: Path) -> None: + """The 1.0 sample passes; the warnings are the missing version, bag and pointcloud map + directories, and hashes not verified (opt-in).""" + result = sanity_check(_copy_v1(tmp_path).as_posix()) + assert result.is_passed() + warnings = {report.id for report in result.reports if not report.is_passed(strict=True)} + assert warnings == {"STR001", "STR005", "STR009", "STR012"} + + +def test_str010_unsupported_version(tmp_path: Path) -> None: + """An unsupported schema version fails, and the other 1.0 rules are skipped.""" + data_root = _copy_v1(tmp_path) + _edit_json(data_root / "manifest.json", lambda m: {**m, "schema_version": "2.0.0"}) + reports = _reports(data_root) + assert reports["STR010"].is_failed() + assert reports["FMT020"].is_failed() + assert reports["STR011"].is_skipped() and reports["FMT021"].is_skipped() + + +def test_str011_missing_file(tmp_path: Path) -> None: + data_root = _copy_v1(tmp_path) + (data_root / "map" / "lanelet2_map.osm").unlink() + assert "STR011" in _failed(_reports(data_root)) + + +def test_str012_checksum(tmp_path: Path) -> None: + """Hashes are verified only with `checksum`; a changed file then fails.""" + data_root = _copy_v1(tmp_path) + with (data_root / "map" / "lanelet2_map.osm").open("a") as f: + f.write(" ") + assert _reports(data_root)["STR012"].severity == "WARNING" + report = _reports(data_root, checksum=True)["STR012"] + assert report.is_failed() and "lanelet2_map.osm" in report.reasons[0] + + +def test_str013_unlisted_file(tmp_path: Path) -> None: + data_root = _copy_v1(tmp_path) + (data_root / "data" / "extra.txt").write_text("not listed") + report = _reports(data_root)["STR013"] + assert not report.is_passed(strict=True) and "data/extra.txt" in report.reasons[0] + + +def test_str014_required_table(tmp_path: Path) -> None: + """An unlisted required table fails STR014, and its format rule is skipped.""" + data_root = _copy_v1(tmp_path) + _edit_json( + data_root / "manifest.json", + lambda m: {**m, "tables": {k: v for k, v in m["tables"].items() if k != "vehicle"}}, + ) + reports = _reports(data_root) + assert reports["STR014"].is_failed() and "vehicle" in reports["STR014"].reasons[0] + assert reports["FMT027"].is_skipped() + + +def test_str015_identity(tmp_path: Path) -> None: + data_root = tmp_path / "other_id" + shutil.copytree(SAMPLE_V1, data_root) + report = _reports(data_root)["STR015"] + assert not report.is_passed(strict=True) and "'sample'" in report.reasons[0] + + +@pytest.mark.parametrize( + ("table", "rule", "change"), + [ + ("sensor", "FMT024", lambda records: [{**records[0], "index": 300}, *records[1:]]), + ("sample", "FMT021", lambda records: [{**records[0], "prev": ""}, *records[1:]]), + ], +) +def test_fmt_invalid_record(tmp_path: Path, table: str, rule: str, change) -> None: + """A record that does not build into its 1.0 class fails its table's format rule.""" + data_root = _copy_v1(tmp_path) + path = data_root / "annotation" / f"{table}.json" + token = json.loads(path.read_text())[0]["token"] + _edit_json(path, change) + report = _reports(data_root)[rule] + assert report.is_failed() and token in report.reasons[0] + + +def test_fmt_unreadable_table(tmp_path: Path) -> None: + data_root = _copy_v1(tmp_path) + (data_root / "annotation" / "scene.json").write_text("not json") + report = _reports(data_root)["FMT025"] + assert report.is_failed() and "cannot be read" in report.reasons[0] diff --git a/tests/sanity/test_v1_record_checkers.py b/tests/sanity/test_v1_record_checkers.py new file mode 100644 index 00000000..4bb40b5b --- /dev/null +++ b/tests/sanity/test_v1_record_checkers.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from t4_devkit.sanity import sanity_check + +SAMPLE_V1 = Path(__file__).parents[1] / "sample" / "t4dataset_v1" + + +def _copy_v1(tmp_path: Path) -> Path: + data_root = tmp_path / "sample" # named as the sample's `dataset_id` + shutil.copytree(SAMPLE_V1, data_root) + return data_root + + +def _edit(data_root: Path, name: str, change) -> None: + """Edit a table (or `manifest`) of the copy in place.""" + path = data_root / ("manifest.json" if name == "manifest" else f"annotation/{name}.json") + path.write_text(json.dumps(change(json.loads(path.read_text())))) + + +def _set(records: list[dict], position: int, **fields) -> list[dict]: + records[position].update(fields) + return records + + +def _reports(data_root: Path, **kwargs) -> dict: + return {report.id: report for report in sanity_check(data_root.as_posix(), **kwargs).reports} + + +def _second_instance(records: list[dict]) -> list[dict]: + return [*_set(records, 0, index=5), {**records[0], "token": "instance_other"}] + + +def _add_label_column(records: list[dict]) -> list[dict]: + records[1]["lidar"]["fields"].append({"name": "label", "dtype": "uint8"}) + return records + + +def _capabilities(change): + def edit(manifest: dict) -> dict: + change(manifest["capabilities"]) + return manifest + + return edit + + +# (rule, table, change): each change breaks only what the rule checks. +CASES = [ + ("REC008", "sensor", lambda r: []), + ("REC009", "sensor", lambda r: _set(r, 1, index=0)), + ("REC010", "instance", _second_instance), + ("REC011", "sensor", _add_label_column), + ("REC012", "scene", lambda r: _set(r, 0, nbr_samples=5)), + ("REC013", "instance", lambda r: _set(r, 0, nbr_annotations=2)), + ("REC014", "object_ann", lambda r: _set(r, 1, instance_token="instance_car")), + ("REC015", "object_ann", lambda r: _set(r, 0, orientation=1.0)), + ("REC016", "sample_annotation", lambda r: _set(r, 0, confidence=0.5)), + ("REC017", "sample_annotation", lambda r: _set(r, 1, provenance=None)), + ("REC018", "sample", lambda r: _set(r, 0, has_key_frame=False)), + ("REC019", "sensor", lambda r: (r[0]["camera"]["video"].update(frame_count=3), r)[1]), + ("REC020", "sample_data", lambda r: _set(r, 0, fileformat="jpg")), + ("REC021", "sample", lambda r: _set(r, 1, timestamp=r[0]["timestamp"] - 1)), + ("REC022", "ego_pose", lambda r: _set(r, 0, timestamp=1_704_067_200)), + ("REC023", "manifest", _capabilities(lambda c: c["modalities"]["camera"].append("CAM_BACK"))), + ("REC024", "manifest", _capabilities(lambda c: c.update(route=False))), + ("REC025", "vehicle", lambda r: _set(r, 0, model="unknown")), + ("REC026", "route", lambda r: _set(r, 0, valid_until=r[0]["valid_from"])), + ("REC027", "sensor", lambda r: _set(r, 0, rotation=[2.0, 0.0, 0.0, 0.0])), + ("REC028", "sample_data", lambda r: _set(r, 0, is_key_frame=False)), +] + + +def test_v1_sample_passes_record_rules() -> None: + reports = _reports(SAMPLE_V1) + assert all(reports[rule].is_passed(strict=True) for rule, *_ in CASES) + + +@pytest.mark.parametrize(("rule", "table", "change"), CASES, ids=[case[0] for case in CASES]) +def test_record_rule_fails(tmp_path: Path, rule: str, table: str, change) -> None: + data_root = _copy_v1(tmp_path) + _edit(data_root, table, change) + report = _reports(data_root)[rule] + assert not report.is_passed(strict=True), report + assert report.reasons + + +@pytest.mark.parametrize( + ("rule", "table", "change"), + [ + ("REC012", "scene", lambda r: _set(r, 0, nbr_samples=5)), + ("REC013", "instance", lambda r: _set(r, 0, nbr_annotations=1, last_annotation_token=None)), + ], +) +def test_record_rule_fix(tmp_path: Path, rule: str, table: str, change) -> None: + """`fix` recomputes the values and updates the table's hash in `manifest.json`.""" + data_root = _copy_v1(tmp_path) + _edit(data_root, table, change) + assert _reports(data_root, fix=True)[rule].fixed + reports = _reports(data_root, checksum=True) + assert reports[rule].is_passed(strict=True) + assert reports["STR012"].is_passed(strict=True) + + +def _write_labels(data_root: Path, frames: list[list[int | None] | None]) -> None: + """Add a `label` column to the LiDAR file and `sensor.json`; `None` = no labels.""" + path = data_root / "data/LIDAR_TOP/LIDAR_TOP.parquet" + parquet = pq.ParquetFile(path) + schema = parquet.schema_arrow.append(pa.field("label", pa.uint8())) + with pq.ParquetWriter(path.with_suffix(".tmp"), schema) as writer: + for i, labels in enumerate(frames): + table = parquet.read_row_group(i) + column = ( + pa.nulls(table.num_rows, pa.uint8()) + if labels is None + else pa.array(labels, pa.uint8()) + ) + writer.write_table(table.append_column("label", column)) + path.with_suffix(".tmp").replace(path) + _edit(data_root, "sensor", _add_label_column) + + +def test_rec029_point_labels(tmp_path: Path) -> None: + """Category 0 must be `unpainted`, and a frame's labels are all null or none null.""" + data_root = _copy_v1(tmp_path) + _write_labels(data_root, [[0] * 8, [1] * 7 + [None], None]) + _edit(data_root, "category", lambda r: _set(_set(r, 0, index=1), 1, index=0)) + + reasons = _reports(data_root)["REC029"].reasons + assert "not 'unpainted'" in reasons[0] + assert "frames [1]" in reasons[1] + + _write_labels(_copy_v1(tmp_path / "ok"), [[0] * 8, [1] * 8, None]) + data_root = tmp_path / "ok" / "sample" + _edit(data_root, "category", lambda r: _set(_set(r, 0, index=1), 1, index=0, name="unpainted")) + assert _reports(data_root)["REC029"].is_passed(strict=True) diff --git a/tests/sanity/test_v1_reference_checkers.py b/tests/sanity/test_v1_reference_checkers.py new file mode 100644 index 00000000..2d845564 --- /dev/null +++ b/tests/sanity/test_v1_reference_checkers.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from t4_devkit.sanity import sanity_check + +SAMPLE_V1 = Path(__file__).parents[1] / "sample" / "t4dataset_v1" +LIDAR_FILE = "data/LIDAR_TOP/LIDAR_TOP.parquet" + + +def _copy_v1(tmp_path: Path) -> Path: + data_root = tmp_path / "sample" # named as the sample's `dataset_id` + shutil.copytree(SAMPLE_V1, data_root) + return data_root + + +def _edit(data_root: Path, name: str, change) -> None: + path = data_root / "annotation" / f"{name}.json" + path.write_text(json.dumps(change(json.loads(path.read_text())))) + + +def _set(records: list[dict], position: int, **fields) -> list[dict]: + records[position].update(fields) + return records + + +def _lidar(records: list[dict], **fields) -> list[dict]: + """Edit the `lidar` block of the LiDAR sensor.""" + records[1]["lidar"].update(fields) + return records + + +def _add_label_column(data_root: Path) -> None: + """Add a `label` column whose values no category has.""" + path = data_root / LIDAR_FILE + parquet = pq.ParquetFile(path) + with pq.ParquetWriter( + path.with_suffix(".tmp"), parquet.schema_arrow.append(pa.field("label", pa.uint8())) + ) as writer: + for i in range(parquet.num_row_groups): + table = parquet.read_row_group(i) + writer.write_table( + table.append_column("label", pa.array([7] * table.num_rows, pa.uint8())) + ) + path.with_suffix(".tmp").replace(path) + + +def _reports(data_root: Path) -> dict: + return {report.id: report for report in sanity_check(data_root.as_posix()).reports} + + +MISSING = "0" * 32 + +# (rule, table, change): each change breaks what the rule checks. +CASES = [ + ("REF015", "sample_data", lambda r: _set(r, 0, sensor_token=MISSING)), + ("REF016", "log", lambda r: _set(r, 0, vehicle_token=MISSING)), + ("REF017", "log", lambda r: _set(r, 0, map_token=None)), + ("REF018", "sample_annotation", lambda r: _set(r, 0, visibility_token=MISSING)), + ("REF019", "object_ann", lambda r: _set(r, 0, category_token=MISSING)), + ("REF020", "keypoint", lambda r: _set(r, 0, category_tokens=[MISSING])), + ("REF021", "traffic_light", lambda r: _set(r, 0, object_ann_token=MISSING)), + ("REF022", "route", lambda r: _set(r, 0, map_token=MISSING)), + ("REF023", "scenario", lambda r: _set(r, 0, log_token=MISSING)), + ("REF024", "caption", lambda r: _set(r, 0, scenario_token=MISSING)), + ("REF025", "sensor", lambda r: _lidar(r, source_channels=["CAM_FRONT"])), + ("REF107", "sample", lambda r: _set(r, 0, next=r[2]["token"])), + ("REF108", "sample_data", lambda r: _set(r, 0, next=r[3]["token"])), + ("REF204", "sample_data", lambda r: _set(r, 0, filename="data/CAM_FRONT/other.mp4")), + ("REF205", "sensor", lambda r: (r[1]["lidar"]["fields"][3].update(dtype="float64"), r)[1]), + ( + "REF206", + "sample_data", + lambda r: [{**x, "num_points": 99} if x["num_points"] else x for x in r], + ), + ("REF207", "map", lambda r: _set(r, 0, sha256="0" * 64)), + ("REF302", "instance", lambda r: _set(r, 0, map_element={"id": 99999, "type": "way"})), + ("REF303", "traffic_light", lambda r: _set(r, 0, regulatory_element_id=99999)), + ( + "REF304", + "route", + lambda r: _set( + r, 0, segments=[{"preferred_primitive_id": 99999, "primitive_ids": [99999]}] + ), + ), + ("REF305", "sensor", lambda r: _lidar(r, source_channels=["LIDAR_TOP"])), +] + + +def test_v1_sample_passes_reference_rules() -> None: + reports = _reports(SAMPLE_V1) + rules = {rule for rule, *_ in CASES} | {"REF306"} + assert all(reports[rule].is_passed(strict=True) for rule in rules) + + +@pytest.mark.parametrize(("rule", "table", "change"), CASES, ids=[case[0] for case in CASES]) +def test_reference_rule_fails(tmp_path: Path, rule: str, table: str, change) -> None: + data_root = _copy_v1(tmp_path) + _edit(data_root, table, change) + report = _reports(data_root)[rule] + assert report.is_failed(), report + + +def test_ref306_point_labels(tmp_path: Path) -> None: + """A LiDAR `label` value that no category index has fails.""" + data_root = _copy_v1(tmp_path) + _add_label_column(data_root) + report = _reports(data_root)["REF306"] + assert report.is_failed() and "[7]" in report.reasons[0]