feat(sanity): T4Dataset 1.0 sanity rules

docs/cli/t4sanity.md

@@ -1,4 +1,4 @@
-`t4sanity` performs sanity checks on T4 datasets, reporting any issues regarding the [dataset requirements](../schema/requirement.md).
+`t4sanity` performs sanity checks on T4 datasets, reporting any issues regarding the dataset requirements: [0.x](../schema/requirement.md) or [1.0](../schema/v1/requirement.md), chosen by the dataset (1.0 if it has a `manifest.json`).
 
 ```shell
 $ t4sanity -h
@@ -15,6 +15,7 @@ $ t4sanity -h
 │ --exclude             -e       TEXT  Exclude specific rules or rule groups. [default: None]                           │
 │ --strict              -s             Indicates whether warnings are treated as failures.                              │
 │ --fix                 -f             Attempt to fix the issues reported by the sanity check.                          │
+│ --checksum            -c             Verify the SHA-256 of every file listed in manifest.json (1.0 datasets).          │
 │ --install-completion                 Install completion for the current shell.                                        │
 │ --show-completion                    Show completion for the current shell, to copy it or customize the installation. │
 │ --help                -h             Show this message and exit.                                                      │

t4_devkit/cli/sanity.py

@@ -45,8 +45,17 @@ def main(
     fix: bool = typer.Option(
         False, "-f", "--fix", help="Attempt to fix the issues reported by the sanity check."
     ),
+    checksum: bool = typer.Option(
+        False,
+        "-c",
+        "--checksum",
+        help="Verify the SHA-256 of every file listed in manifest.json (1.0 datasets). "
+        "Reads every file, including videos and rosbags.",
+    ),
 ) -> None:
-    result = sanity_check(data_root=data_root, revision=revision, excludes=excludes, fix=fix)
+    result = sanity_check(
+        data_root=data_root, revision=revision, excludes=excludes, fix=fix, checksum=checksum
+    )
 
     print_sanity_result(result, strict=strict)
 

t4_devkit/sanity/checker.py

@@ -6,17 +6,33 @@
 
 from returns.maybe import Maybe, Nothing, Some
 
-from .result import make_report, make_skipped
+from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS
+
+from .result import Reason, make_report, make_skipped
 
 if TYPE_CHECKING:
+    from t4_devkit.schema.v1 import SchemaName as V1SchemaName
+
     from .context import SanityContext
-    from .result import Reason, Report
+    from .result import Report
 
 
 RuleID = NewType("RuleID", str)
 RuleName = NewType("RuleName", str)
 
 
+def record_name(table: V1SchemaName) -> str:
+    """Return the class name of a 1.0 table's records (e.g. "SampleAnnotation"), for reasons."""
+    return V1_SCHEMAS.get(table).__name__
+
+
+class SchemaVersion(str, Enum):
+    """Dataset schema versions a rule can apply to."""
+
+    V0 = "0.x"
+    V1 = "1.0"
+
+
 class Severity(str, Enum):
     """Severity levels for sanity checkers."""
 
@@ -33,12 +49,17 @@ def is_error(self) -> bool:
 
 
 class Checker(ABC):
-    """Base class for sanity checkers."""
+    """Base class for sanity checkers.
+
+    Attributes:
+        versions (tuple[SchemaVersion, ...]): Schema versions the rule applies to; 0.x by default.
+    """
 
     id: RuleID
     name: RuleName
     description: str
     severity: Severity
+    versions: tuple[SchemaVersion, ...] = (SchemaVersion.V0,)
 
     def __call__(self, context: SanityContext, fix: bool = False) -> Report:
         """Run the checker and return a report.
@@ -94,3 +115,36 @@ def fix(self, context: SanityContext) -> bool:
             True if the issue was fixed, False otherwise.
         """
         return False
+
+
+class ManifestChecker(Checker):
+    """Base class for 1.0 rules that read `manifest.json`: skipped when it cannot be read or is
+    not valid (reported by STR010 and FMT020)."""
+
+    versions = (SchemaVersion.V1,)
+
+    def can_skip(self, context: SanityContext) -> Maybe[Reason]:
+        match context.manifest:
+            case Some(_):
+                return Nothing
+        return Maybe.from_value(Reason("No valid 'manifest.json' (see STR010, FMT020)"))
+
+
+class RecordsChecker(ManifestChecker):
+    """Base class for 1.0 rules on records: skipped unless every table they read is listed (or
+    optional) and builds (reported by STR014 and the format rules).
+
+    Attributes:
+        tables (tuple[V1SchemaName, ...]): The tables the rule reads.
+    """
+
+    tables: tuple[V1SchemaName, ...]
+
+    def can_skip(self, context: SanityContext) -> Maybe[Reason]:
+        skip = super().can_skip(context)
+        if skip != Nothing:
+            return skip
+        invalid = [table.value for table in self.tables if context.load_records(table) == Nothing]
+        if invalid:
+            return Maybe.from_value(Reason(f"Tables not listed or not valid: {', '.join(invalid)}"))
+        return Nothing

t4_devkit/sanity/context.py

@@ -1,30 +1,50 @@
 from __future__ import annotations
 
+import hashlib
+from functools import cached_property
 from pathlib import Path
 
-from attrs import define
-from returns.maybe import Maybe, Some
+from attrs import define, field
+from returns.converters import result_to_maybe
+from returns.maybe import Maybe, Nothing, Some
 from returns.pipeline import is_successful
+from returns.result import safe
 from typing_extensions import TYPE_CHECKING, Self
 
 from t4_devkit import DBMetadata
+from t4_devkit.lanelet import LaneletParser
 from t4_devkit.schema import SchemaName
+from t4_devkit.schema.v1 import MANIFEST_FILENAME, SCHEMAS, Manifest
+from t4_devkit.schema.v1 import SchemaName as V1SchemaName
 
-from .safety import load_metadata_safe, save_json_safe
+from .checker import SchemaVersion
+from .safety import load_json_safe, load_manifest_safe, load_metadata_safe, save_json_safe
 
 if TYPE_CHECKING:
     from t4_devkit.typing import PathLike
 
 
-@define
+@define(slots=False)
 class SanityContext:
+    """What the checkers read: the dataset paths, and for 1.0 its manifest and tables, read once.
+
+    Attributes:
+        metadata (Maybe[DBMetadata]): Dataset root, ID and version directory.
+        checksum (bool): Whether to verify the SHA-256 of the files listed in `manifest.json`.
+    """
+
     metadata: Maybe[DBMetadata]
+    checksum: bool = field(default=False, kw_only=True)
+    _tables: dict[V1SchemaName, Maybe[list[dict]]] = field(factory=dict, init=False, repr=False)
+    _records: dict[V1SchemaName, Maybe[list]] = field(factory=dict, init=False, repr=False)
 
     @classmethod
-    def from_path(cls, data_root: PathLike, revision: str | None = None) -> Self:
+    def from_path(
+        cls, data_root: PathLike, revision: str | None = None, *, checksum: bool = False
+    ) -> Self:
         metadata_result = load_metadata_safe(data_root, revision=revision)
         metadata = metadata_result.unwrap() if is_successful(metadata_result) else None
-        return cls(Maybe.from_optional(metadata))
+        return cls(Maybe.from_optional(metadata), checksum=checksum)
 
     @property
     def data_root(self) -> Maybe[Path]:
@@ -66,12 +86,98 @@ def status_json(self) -> Maybe[Path]:
         """Return the path to status JSON file, which is 'status.json'."""
         return self.metadata.map(lambda m: Path(m.data_root).joinpath("status.json"))
 
+    @property
+    def manifest_json(self) -> Maybe[Path]:
+        """Return the path to `manifest.json`."""
+        return self.metadata.map(lambda m: Path(m.data_root).joinpath(MANIFEST_FILENAME))
+
+    @property
+    def schema_version(self) -> SchemaVersion:
+        """Return 1.0 if `manifest.json` exists at the dataset root, 0.x otherwise."""
+        match self.manifest_json:
+            case Some(x) if x.exists():
+                return SchemaVersion.V1
+        return SchemaVersion.V0
+
+    @cached_property
+    def raw_manifest(self) -> Maybe[dict]:
+        """Return `manifest.json` as read, or Nothing if it cannot be read."""
+        match self.manifest_json:
+            case Some(x) if x.exists():
+                result = load_json_safe(x)
+                return Some(result.unwrap()) if is_successful(result) else Nothing
+        return Nothing
+
+    @cached_property
+    def manifest(self) -> Maybe[Manifest]:
+        """Return the manifest, or Nothing if it cannot be read or is not valid."""
+        match self.raw_manifest:
+            case Some(data):
+                return result_to_maybe(load_manifest_safe(data))
+        return Nothing
+
+    def to_table_file(self, table: V1SchemaName) -> Maybe[Path]:
+        """Return the path of a 1.0 table file listed in `manifest.json`, or Nothing if unlisted."""
+        match self.manifest, self.data_root:
+            case Some(manifest), Some(data_root) if table.value in manifest.tables:
+                return Some(data_root.joinpath(manifest.tables[table.value].path))
+        return Nothing
+
+    def load_table(self, table: V1SchemaName) -> Maybe[list[dict]]:
+        """Return the raw records of a 1.0 table as read (read once), or Nothing if unlisted or
+        unreadable."""
+        if table not in self._tables:
+            match self.to_table_file(table):
+                case Some(path):
+                    self._tables[table] = result_to_maybe(load_json_safe(path))
+                case _:
+                    self._tables[table] = Nothing
+        return self._tables[table]
+
+    def load_records(self, table: V1SchemaName) -> Maybe[list]:
+        """Return the 1.0 records of a table (built once): empty for an unlisted optional table,
+        Nothing if the table is required and unlisted, unreadable, or has an invalid record
+        (reported by STR014 and the format rules)."""
+        if table not in self._records:
+            self._records[table] = self._build_records(table)
+        return self._records[table]
+
+    @cached_property
+    def lanelet_map(self) -> Maybe[LaneletParser]:
+        """Return the lanelet2 map of the dataset (read once): the file of the first `Map`
+        record, or Nothing if there is none or it cannot be read."""
+        match self.load_records(V1SchemaName.MAP), self.data_root:
+            case Some([map_record, *_]), Some(data_root):
+                return result_to_maybe(_parse_lanelet_safe(data_root.joinpath(map_record.filename)))
+        return Nothing
+
+    def save_table(self, table: V1SchemaName, records: list[dict]) -> bool:
+        """Save the raw records of a 1.0 table, and its new SHA-256 in `manifest.json`.
+
+        Returns:
+            True if both files were written.
+        """
+        filepath = self.to_table_file(table).value_or(None)
+        if filepath is None or not is_successful(save_json_safe(records, filepath)):
+            return False
+        manifest = self.raw_manifest.unwrap()
+        manifest["tables"][table.value]["sha256"] = hashlib.sha256(
+            filepath.read_bytes()
+        ).hexdigest()
+        saved = is_successful(save_json_safe(manifest, self.manifest_json.unwrap()))
+        # read again on next use
+        self._tables.pop(table, None)
+        self._records.pop(table, None)
+        return saved
+
     def to_schema_file(self, schema: SchemaName) -> Maybe[Path]:
         """Convert schema name to file path, which is <data_root>/annotation/<schema_name>.json."""
         return self.annotation_dir.map(lambda ann: ann.joinpath(schema.filename))
 
     def save_records(self, schema: SchemaName, records: list[dict]) -> bool:
-        """Save schema data to file."""
+        """Save schema data to file (for 1.0, also its SHA-256 in `manifest.json`)."""
+        if self.schema_version == SchemaVersion.V1:
+            return self.save_table(V1SchemaName(schema.value), records)
         match self.to_schema_file(schema):
             case Some(filepath):
                 result = save_json_safe(records, filepath)
@@ -81,3 +187,24 @@ def save_records(self, schema: SchemaName, records: list[dict]) -> bool:
                     print(f"Error saving schema: {result.failure()}")
                     return False
         return False
+
+    def _build_records(self, table: V1SchemaName) -> Maybe[list]:
+        match self.manifest:
+            case Some(manifest):
+                if table.value not in manifest.tables:
+                    return Some([]) if table.is_optional() else Nothing
+                match self.load_table(table):
+                    case Some(records):
+                        return result_to_maybe(_build_safe(table, records))
+        return Nothing
+
+
+@safe
+def _build_safe(table: V1SchemaName, records: list[dict]) -> list:
+    """Build the 1.0 records of a table; fails on the first invalid record."""
+    return [SCHEMAS.get(table).from_dict(record) for record in records]
+
+
+@safe
+def _parse_lanelet_safe(path: Path) -> LaneletParser:
+    return LaneletParser(path.as_posix(), verbose=False)

t4_devkit/sanity/format/__init__.py

@@ -19,3 +19,24 @@
 from .fmt017 import *  # noqa
 from .fmt018 import *  # noqa
 from .fmt019 import *  # noqa
+from .fmt020 import *  # noqa
+from .fmt021 import *  # noqa
+from .fmt022 import *  # noqa
+from .fmt023 import *  # noqa
+from .fmt024 import *  # noqa
+from .fmt025 import *  # noqa
+from .fmt026 import *  # noqa
+from .fmt027 import *  # noqa
+from .fmt028 import *  # noqa
+from .fmt029 import *  # noqa
+from .fmt030 import *  # noqa
+from .fmt031 import *  # noqa
+from .fmt032 import *  # noqa
+from .fmt033 import *  # noqa
+from .fmt034 import *  # noqa
+from .fmt035 import *  # noqa
+from .fmt036 import *  # noqa
+from .fmt037 import *  # noqa
+from .fmt038 import *  # noqa
+from .fmt039 import *  # noqa
+from .fmt040 import *  # noqa

t4_devkit/sanity/format/base.py

@@ -6,8 +6,10 @@
 from returns.pipeline import is_successful
 
 from t4_devkit.schema import SCHEMAS, SchemaName
+from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS
+from t4_devkit.schema.v1 import SchemaName as V1SchemaName
 
-from ..checker import Checker
+from ..checker import Checker, ManifestChecker
 from ..result import Reason
 from ..safety import load_json_safe, load_schema_safe
 
@@ -55,3 +57,43 @@ def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason] | No
         if not is_successful(conversion):
             failures.append(Reason(f"[{schema.name}] {record['token']}: {conversion.failure()}"))
     return failures if failures else None
+
+
+class TableFieldChecker(ManifestChecker):
+    """Base class for 1.0 format checkers: every record of a table builds into its 1.0 class.
+
+    Attributes:
+        table (V1SchemaName): The 1.0 table to check.
+    """
+
+    table: V1SchemaName
+
+    def can_skip(self, context: SanityContext) -> Maybe[Reason]:
+        skip = super().can_skip(context)
+        if skip != Nothing:
+            return skip
+        if (
+            not self.table.is_optional()
+            and self.table.value not in context.manifest.unwrap().tables
+        ):
+            return Maybe.from_value(Reason(f"'{self.table.value}' not listed (see STR014)"))
+        return Nothing
+
+    def check(self, context: SanityContext) -> list[Reason] | None:
+        if self.table.value not in context.manifest.unwrap().tables:
+            return None  # optional table not present
+        match context.load_table(self.table):
+            case Some(records):
+                return _build_v1_records(self.table, records)
+        return [Reason(f"'{context.to_table_file(self.table).unwrap()}' cannot be read")]
+
+
+def _build_v1_records(table: V1SchemaName, records: list[dict]) -> list[Reason] | None:
+    module = V1_SCHEMAS.get(table)
+    failures = []
+    for record in records:
+        conversion = load_schema_safe(module, record)
+        if not is_successful(conversion):
+            token = record.get("token") if isinstance(record, dict) else None
+            failures.append(Reason(f"[{table.name}] {token}: {conversion.failure()}"))
+    return failures if failures else None