From 44dcdc935ab8ff836e2fbda629c3c25d0e7845c5 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 15:24:47 +0900 Subject: [PATCH 01/12] feat(common): add t4pack format to pack pointcloud frames into a single file Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 + t4_devkit/common/t4pack.py | 497 ++++++++++++++++++++++++++++++++++++ tests/common/test_t4pack.py | 186 ++++++++++++++ 3 files changed, 685 insertions(+) create mode 100644 t4_devkit/common/t4pack.py create mode 100644 tests/common/test_t4pack.py diff --git a/pyproject.toml b/pyproject.toml index b04a8855..00225c79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "returns>=0.26.0", "pyarrow<24.0.0", # NOTE: pyarrow==24.0.0 is only supported on macOS "pypcd4>=1.4.3,<2", + "zstandard>=0.23.0", ] [dependency-groups] @@ -42,6 +43,7 @@ dev = [ [project.scripts] t4viz = "t4_devkit.cli.visualize:cli" t4sanity = "t4_devkit.cli.sanity:cli" +t4pack = "t4_devkit.cli.pack:cli" [tool.ruff] line-length = 100 diff --git a/t4_devkit/common/t4pack.py b/t4_devkit/common/t4pack.py new file mode 100644 index 00000000..541b614c --- /dev/null +++ b/t4_devkit/common/t4pack.py @@ -0,0 +1,497 @@ +"""t4pack: a container that stores all pointcloud frames of a sensor channel in a single file. + +Format (t4pack v1): + +``` +magic b'T4PACK\\x00\\x01' +blob_0 .. blob_{N-1} zstd(column_0 bytes | column_1 bytes | ...) +zstd(JSON index) name, offset, size, n_points, dtypes per frame + little-endian +magic +``` + +Points are stored column-major, one contiguous plane per column. Float32 columns are +byte-shuffled (dtype code `f4s`): the four bytes of every float are split into four planes, +so the near-constant sign/exponent bytes are grouped together and zstd can match them. +Each frame is compressed independently, so reading one frame decodes only that frame. + +The format is compatible with the e2e-devkit `lidar_pack.py` reader. +""" + +from __future__ import annotations + +import json +import os +import os.path as osp +import struct +from typing import TYPE_CHECKING, Iterable, Sequence + +import numpy as np +import zstandard as zstd +from attrs import define + +if TYPE_CHECKING: + from typing_extensions import Self + + from t4_devkit.typing import NDArrayF32, PathLike + +__all__ = ( + "PACK_SUFFIX", + "PackLayout", + "PackReader", + "PackStats", + "encode_frame", + "decode_frame", + "write_pack", + "pack_directory", + "verify_pack", + "find_pack", +) + +MAGIC = b"T4PACK\x00\x01" +TRAILER = struct.Struct(" Self: + """Return the layout for points with the specified number of float32 features. + + The 1-byte columns (`u1`, `i1`) are only valid when the source values are integers in range, + which is guaranteed by verifying every frame after packing. + Unknown widths are stored as byte-shuffled float32 columns, which is always lossless. + + Args: + num_pts_feats (int): Number of float32 fields in each point. + + Returns: + Self: Layout of columns. + """ + if num_pts_feats == 5: + return cls( + dtypes=("f4s", "f4s", "f4s", "u1", "i1"), + fields=("x", "y", "z", "intensity", "ring"), + ) + elif num_pts_feats == 7: + return cls( + dtypes=("f4s", "f4s", "f4s", "u1", "i1", "u1", "f4s"), + fields=("x", "y", "z", "intensity", "channel", "return_type", "timestamp"), + ) + elif num_pts_feats >= 4: + return cls( + dtypes=("f4s",) * num_pts_feats, + fields=("x", "y", "z", "intensity") + + tuple(f"feature{i}" for i in range(4, num_pts_feats)), + ) + else: + raise ValueError(f"Expected num_pts_feats >= 4, but got {num_pts_feats}") + + +@define +class PackStats: + """A dataclass to represent the result of packing frames. + + Attributes: + num_frames (int): Number of packed frames. + raw_bytes (int): Total size of the source frames in bytes. + packed_bytes (int): Size of the pack file in bytes. + """ + + num_frames: int + raw_bytes: int + packed_bytes: int + + @property + def ratio(self) -> float: + """Return the compression ratio. + + Returns: + float: `raw_bytes / packed_bytes`. + """ + return self.raw_bytes / self.packed_bytes if self.packed_bytes else 0.0 + + +def encode_frame(frame: NDArrayF32, dtypes: Sequence[str]) -> bytes: + """Serialize one frame into an uncompressed blob. + + Args: + frame (NDArrayF32): Points in the shape of (N, len(dtypes)). + dtypes (Sequence[str]): t4pack dtype code of each column. + + Returns: + bytes: Serialized column-major blob. + """ + frame = np.asarray(frame, dtype=np.float32) + if frame.ndim != 2 or frame.shape[1] != len(dtypes): + raise ValueError(f"Expected frame shape is (N, {len(dtypes)}), but got {frame.shape}") + + parts = [] + for column, dtype in enumerate(dtypes): + values = frame[:, column] + if dtype == "f4s": + planes = np.ascontiguousarray(values, np.float32).view(np.uint8).reshape(-1, 4) + parts.append(np.ascontiguousarray(planes.T).tobytes()) + elif dtype == "u1": + parts.append(values.astype(np.uint8).tobytes()) + elif dtype == "i1": + parts.append(values.astype(np.int8).tobytes()) + else: + raise ValueError(f"Unknown t4pack dtype at column {column}: {dtype}") + return b"".join(parts) + + +def decode_frame(raw: bytes, dtypes: Sequence[str], num_points: int) -> NDArrayF32: + """Deserialize an uncompressed blob into one frame. This is the inverse of `encode_frame`. + + Args: + raw (bytes): Serialized column-major blob. + dtypes (Sequence[str]): t4pack dtype code of each column. + num_points (int): Number of points in the frame. + + Returns: + NDArrayF32: Points in the shape of (num_points, len(dtypes)). + """ + out = np.empty((num_points, len(dtypes)), dtype=np.float32) + pos = 0 + for column, dtype in enumerate(dtypes): + if dtype == "f4s": + planes = np.frombuffer(raw, np.uint8, 4 * num_points, pos).reshape(4, num_points) + out[:, column] = np.ascontiguousarray(planes.T).reshape(-1).view(np.float32) + pos += 4 * num_points + elif dtype == "u1": + out[:, column] = np.frombuffer(raw, np.uint8, num_points, pos) + pos += num_points + elif dtype == "i1": + out[:, column] = np.frombuffer(raw, np.int8, num_points, pos) + pos += num_points + else: + raise ValueError(f"Unknown t4pack dtype at column {column}: {dtype}") + + if pos != len(raw): + raise ValueError(f"Frame has {len(raw) - pos} trailing bytes") + return out + + +def write_pack( + filepath: PathLike, + frames: Iterable[NDArrayF32], + layout: PackLayout, + names: Sequence[str] | None = None, + level: int = 19, +) -> int: + """Write frames into a single pack file. + + The pack is written to `.tmp` first and renamed, + so an interrupted write never leaves a partial pack in place. + + Args: + filepath (PathLike): Path to the output pack file. + frames (Iterable[NDArrayF32]): Frames in the shape of (N_i, len(layout.dtypes)). + layout (PackLayout): Column layout of points. + names (Sequence[str] | None, optional): File name of each frame, used to look up frames. + If None, `.pcd.bin` is used. + level (int, optional): zstd compression level. + + Returns: + int: Number of written frames. + """ + filepath = os.fspath(filepath) + tmp_filepath = f"{filepath}.tmp" + compressor = zstd.ZstdCompressor(level=level) + entries = [] + try: + with open(tmp_filepath, "wb") as f: + f.write(MAGIC) + for index, frame in enumerate(frames): + if names is not None and index >= len(names): + raise ValueError(f"Got more frames than {len(names)} names") + frame = np.asarray(frame, dtype=np.float32) + blob = compressor.compress(encode_frame(frame, layout.dtypes)) + entries.append( + { + "name": f"{index:05d}{FRAME_SUFFIX}" if names is None else names[index], + "offset": f.tell(), + "size": len(blob), + "n_points": int(frame.shape[0]), + "dtypes": list(layout.dtypes), + } + ) + f.write(blob) + + if names is not None and len(names) != len(entries): + raise ValueError(f"Got {len(names)} names for {len(entries)} frames") + + index_blob = compressor.compress( + json.dumps( + { + "format": "t4pack", + "version": 1, + "n_frames": len(entries), + "fields": list(layout.fields), + "frames": entries, + }, + separators=(",", ":"), + ).encode() + ) + index_offset = f.tell() + f.write(index_blob) + f.write(TRAILER.pack(index_offset, len(index_blob))) + f.write(MAGIC) + os.replace(tmp_filepath, filepath) + finally: + if osp.exists(tmp_filepath): + os.remove(tmp_filepath) + + return len(entries) + + +class PackReader: + """Random-access reader of a pack file. + + This is not thread-safe, because it holds one file descriptor and one zstd decompressor. + Open one reader per thread. + + Examples: + >>> from t4_devkit.common.t4pack import PackReader + >>> with PackReader("data/LIDAR_CONCAT.pack") as reader: + ... points = reader.read_frame_by_name("00000.pcd.bin") + """ + + def __init__(self, filepath: PathLike) -> None: + """Open a pack file and load its frame index. + + Args: + filepath (PathLike): Path to the pack file. + """ + self.filepath = os.fspath(filepath) + self._fd = os.open(self.filepath, os.O_RDONLY) + self._dctx = zstd.ZstdDecompressor() + try: + size = os.fstat(self._fd).st_size + trailer_size = TRAILER.size + len(MAGIC) + trailer = os.pread(self._fd, trailer_size, max(size - trailer_size, 0)) + if ( + len(trailer) != trailer_size + or trailer[TRAILER.size :] != MAGIC + or os.pread(self._fd, len(MAGIC), 0) != MAGIC + ): + raise ValueError(f"Not a t4pack file: {self.filepath}") + + index_offset, index_size = TRAILER.unpack(trailer[: TRAILER.size]) + index = json.loads(self._dctx.decompress(os.pread(self._fd, index_size, index_offset))) + if index.get("format") != "t4pack" or index.get("version") != 1: + raise ValueError( + f"Unsupported pack format: {index.get('format')}/{index.get('version')}" + ) + + self.frames: list[dict] = index["frames"] + self.fields: list[str] | None = index.get("fields") + if int(index["n_frames"]) != len(self.frames): + raise ValueError(f"n_frames does not match the frame index: {self.filepath}") + self._name_to_index = {frame["name"]: i for i, frame in enumerate(self.frames)} + except Exception: + self.close() + raise + + def __len__(self) -> int: + return len(self.frames) + + def __enter__(self) -> Self: + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def __del__(self) -> None: + self.close() + + @property + def names(self) -> list[str]: + """Return the file name of each frame. + + Returns: + list[str]: List of frame names in the stored order. + """ + return [frame["name"] for frame in self.frames] + + def close(self) -> None: + """Close the file descriptor.""" + fd = getattr(self, "_fd", -1) + if fd >= 0: + os.close(fd) + self._fd = -1 + + def read_frame(self, index: int) -> NDArrayF32: + """Decode the frame at the specified index. + + Args: + index (int): Index of the frame. + + Returns: + NDArrayF32: Points in the shape of (N, num_columns). + """ + if not 0 <= index < len(self.frames): + raise IndexError(f"Frame index {index} is out of range [0, {len(self.frames)})") + + frame = self.frames[index] + blob = os.pread(self._fd, int(frame["size"]), int(frame["offset"])) + return decode_frame( + self._dctx.decompress(blob), tuple(frame["dtypes"]), int(frame["n_points"]) + ) + + def read_frame_by_name(self, name: str) -> NDArrayF32: + """Decode the frame with the specified file name. + + Args: + name (str): File name of the frame, e.g. `00000.pcd.bin`. + + Returns: + NDArrayF32: Points in the shape of (N, num_columns). + """ + if name not in self._name_to_index: + raise KeyError(f"Frame '{name}' is not found in {self.filepath}") + return self.read_frame(self._name_to_index[name]) + + +def _load_frame(filepath: str, num_pts_feats: int) -> NDArrayF32: + scan = np.fromfile(filepath, dtype=np.float32) + if scan.size % num_pts_feats != 0: + raise ValueError(f"{scan.size} floats is not a multiple of {num_pts_feats}: {filepath}") + return scan.reshape((-1, num_pts_feats)) + + +def _list_frames(directory: str) -> list[str]: + return sorted(name for name in os.listdir(directory) if name.endswith(FRAME_SUFFIX)) + + +def pack_directory( + directory: PathLike, + num_pts_feats: int = 5, + level: int = 19, + overwrite: bool = False, +) -> PackStats | None: + """Pack all `.pcd.bin` frames in a directory into `.pack`. + + Every frame is verified bit-exact against its source before the pack is put in place, + so a pack that exists is always complete. Source frames are not deleted. + + Args: + directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. + num_pts_feats (int, optional): Number of float32 fields in each point. + level (int, optional): zstd compression level. + overwrite (bool, optional): Whether to overwrite an existing pack. + + Returns: + PackStats | None: Result of packing, or None if a valid pack already exists. + """ + directory = os.fspath(directory).rstrip(os.sep) + pack_filepath = directory + PACK_SUFFIX + + if osp.exists(pack_filepath) and not overwrite: + try: + with PackReader(pack_filepath): + return None + except Exception: + pass # broken pack, rewrite it + + names = _list_frames(directory) + if not names: + raise FileNotFoundError(f"No {FRAME_SUFFIX} frames in {directory}") + + filepaths = [osp.join(directory, name) for name in names] + layout = PackLayout.from_num_pts_feats(num_pts_feats) + + # write to a temporary file, verify it, and then rename it to the final path + tmp_filepath = pack_filepath + ".unverified" + try: + num_frames = write_pack( + tmp_filepath, + (_load_frame(filepath, num_pts_feats) for filepath in filepaths), + layout=layout, + names=names, + level=level, + ) + mismatches = _compare_frames(tmp_filepath, directory, names, num_pts_feats) + if mismatches: + raise ValueError( + f"{len(mismatches)} frames differ after packing {directory}, e.g. {mismatches[:3]}" + ) + os.replace(tmp_filepath, pack_filepath) + finally: + if osp.exists(tmp_filepath): + os.remove(tmp_filepath) + + return PackStats( + num_frames=num_frames, + raw_bytes=sum(osp.getsize(filepath) for filepath in filepaths), + packed_bytes=osp.getsize(pack_filepath), + ) + + +def verify_pack(directory: PathLike, num_pts_feats: int = 5) -> list[str]: + """Verify that every frame in `.pack` equals its source `.pcd.bin` frame. + + Args: + directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. + num_pts_feats (int, optional): Number of float32 fields in each point. + + Returns: + list[str]: Names of the frames that are missing or differ. Empty if the pack is valid. + """ + directory = os.fspath(directory).rstrip(os.sep) + return _compare_frames( + directory + PACK_SUFFIX, directory, _list_frames(directory), num_pts_feats + ) + + +def _compare_frames( + pack_filepath: str, + directory: str, + names: list[str], + num_pts_feats: int, +) -> list[str]: + mismatches = [] + with PackReader(pack_filepath) as reader: + source_names = set(names) + mismatches.extend(name for name in reader.names if name not in source_names) + for name in names: + try: + packed = reader.read_frame_by_name(name) + except KeyError: + mismatches.append(name) + continue + if not np.array_equal(packed, _load_frame(osp.join(directory, name), num_pts_feats)): + mismatches.append(name) + return mismatches + + +def find_pack(filepath: PathLike) -> str | None: + """Return the path to the pack containing the specified frame if it exists. + + For example, the pack of `data/LIDAR_CONCAT/00000.pcd.bin` is `data/LIDAR_CONCAT.pack`. + + Args: + filepath (PathLike): Path to a frame file. + + Returns: + str | None: Path to the pack file, or None if it does not exist. + """ + pack_filepath = osp.dirname(os.fspath(filepath)) + PACK_SUFFIX + return pack_filepath if osp.exists(pack_filepath) else None diff --git a/tests/common/test_t4pack.py b/tests/common/test_t4pack.py new file mode 100644 index 00000000..b0e33152 --- /dev/null +++ b/tests/common/test_t4pack.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from t4_devkit.common.t4pack import ( + PackLayout, + PackReader, + decode_frame, + encode_frame, + find_pack, + pack_directory, + verify_pack, + write_pack, +) + + +def _frame(num_points: int, num_pts_feats: int = 5, seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + frame = rng.normal(0, 30, (num_points, num_pts_feats)).astype(np.float32) + frame[:, 3] = rng.integers(0, 256, num_points) + frame[:, 4] = rng.integers(-1, 128, num_points) + return frame + + +def _write_frames(directory: Path, frames: list[np.ndarray], names: list[str]) -> None: + directory.mkdir(parents=True) + for frame, name in zip(frames, names, strict=True): + frame.tofile(directory / name) + + +@pytest.mark.parametrize( + ("num_pts_feats", "dtypes"), + [ + (5, ("f4s", "f4s", "f4s", "u1", "i1")), + (7, ("f4s", "f4s", "f4s", "u1", "i1", "u1", "f4s")), + (6, ("f4s",) * 6), + ], +) +def test_pack_layout_from_num_pts_feats(num_pts_feats: int, dtypes: tuple[str, ...]) -> None: + layout = PackLayout.from_num_pts_feats(num_pts_feats) + + assert layout.dtypes == dtypes + assert len(layout.fields) == num_pts_feats + assert layout.fields[:4] == ("x", "y", "z", "intensity") + + +def test_pack_layout_rejects_too_few_features() -> None: + with pytest.raises(ValueError): + PackLayout.from_num_pts_feats(3) + + +@pytest.mark.parametrize("num_points", [0, 1, 1000]) +def test_encode_decode_frame_round_trip(num_points: int) -> None: + layout = PackLayout.from_num_pts_feats(5) + frame = _frame(num_points) + + decoded = decode_frame(encode_frame(frame, layout.dtypes), layout.dtypes, num_points) + + assert np.array_equal(decoded, frame) + + +def test_encode_frame_rejects_wrong_width() -> None: + with pytest.raises(ValueError): + encode_frame(np.zeros((4, 4), dtype=np.float32), PackLayout.from_num_pts_feats(5).dtypes) + + +def test_write_and_read_pack(tmp_path: Path) -> None: + filepath = tmp_path / "LIDAR_CONCAT.pack" + layout = PackLayout.from_num_pts_feats(5) + frames = [_frame(500, seed=0), _frame(0, seed=1), _frame(17, seed=2)] + names = ["0.pcd.bin", "1.pcd.bin", "10.pcd.bin"] + + num_frames = write_pack(filepath, frames, layout=layout, names=names, level=1) + + assert num_frames == 3 + assert not (tmp_path / "LIDAR_CONCAT.pack.tmp").exists() + with PackReader(filepath) as reader: + assert len(reader) == 3 + assert reader.names == names + assert reader.fields == list(layout.fields) + for i, frame in enumerate(frames): + assert np.array_equal(reader.read_frame(i), frame) + assert np.array_equal(reader.read_frame_by_name("10.pcd.bin"), frames[2]) + with pytest.raises(IndexError): + reader.read_frame(3) + with pytest.raises(KeyError): + reader.read_frame_by_name("2.pcd.bin") + + +def test_write_pack_names_frames_by_index_by_default(tmp_path: Path) -> None: + filepath = tmp_path / "LIDAR_CONCAT.pack" + frames = [_frame(10, seed=0), _frame(10, seed=1)] + + write_pack(filepath, frames, layout=PackLayout.from_num_pts_feats(5), level=1) + + with PackReader(filepath) as reader: + assert reader.names == ["00000.pcd.bin", "00001.pcd.bin"] + assert np.array_equal(reader.read_frame_by_name("00001.pcd.bin"), frames[1]) + + +def test_write_pack_rejects_mismatched_names(tmp_path: Path) -> None: + filepath = tmp_path / "LIDAR_CONCAT.pack" + + with pytest.raises(ValueError): + write_pack( + filepath, + [_frame(10)], + layout=PackLayout.from_num_pts_feats(5), + names=["00000.pcd.bin", "00001.pcd.bin"], + ) + assert not filepath.exists() + assert not (tmp_path / "LIDAR_CONCAT.pack.tmp").exists() + + +def test_pack_reader_rejects_non_pack_file(tmp_path: Path) -> None: + filepath = tmp_path / "junk.pack" + filepath.write_bytes(b"not a pack at all") + + with pytest.raises(ValueError): + PackReader(filepath) + + +def test_pack_directory_and_verify(tmp_path: Path) -> None: + directory = tmp_path / "data" / "LIDAR_CONCAT" + names = ["00000.pcd.bin", "00001.pcd.bin", "00002.pcd.bin"] + frames = [_frame(100, seed=i) for i in range(len(names))] + _write_frames(directory, frames, names) + + stats = pack_directory(directory, level=1) + + assert stats is not None + assert stats.num_frames == 3 + assert stats.raw_bytes == sum(frame.nbytes for frame in frames) + assert stats.ratio > 1.0 + assert (tmp_path / "data" / "LIDAR_CONCAT.pack").exists() + assert verify_pack(directory) == [] + + # already packed + assert pack_directory(directory, level=1) is None + assert pack_directory(directory, level=1, overwrite=True) is not None + + # a changed source frame is reported + _frame(100, seed=99).tofile(directory / names[1]) + assert verify_pack(directory) == [names[1]] + + +def test_pack_directory_supports_wide_points(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + frame = _frame(100, num_pts_feats=7) + frame[:, 5] = 1.0 + _write_frames(directory, [frame], ["00000.pcd.bin"]) + + pack_directory(directory, num_pts_feats=7, level=1) + + assert verify_pack(directory, num_pts_feats=7) == [] + + +def test_pack_directory_refuses_lossy_pack(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + frame = _frame(100) + frame[0, 3] = 0.5 # intensity is not an integer + _write_frames(directory, [frame], ["00000.pcd.bin"]) + + with pytest.raises(ValueError): + pack_directory(directory, level=1) + assert list(tmp_path.glob("*.pack*")) == [] + + +def test_pack_directory_requires_frames(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + directory.mkdir() + + with pytest.raises(FileNotFoundError): + pack_directory(directory) + + +def test_find_pack(tmp_path: Path) -> None: + filepath = tmp_path / "LIDAR_CONCAT" / "00000.pcd.bin" + + assert find_pack(filepath) is None + + (tmp_path / "LIDAR_CONCAT.pack").touch() + assert find_pack(filepath) == str(tmp_path / "LIDAR_CONCAT.pack") From 87f83afd3ce2a363667aedc609bd07a9b21af580 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 15:24:47 +0900 Subject: [PATCH 02/12] feat(dataclass): read pointcloud frames from pack if .pcd.bin does not exist Co-Authored-By: Claude Opus 5.5 --- t4_devkit/dataclass/pointcloud.py | 28 ++++++++++++++++--- tests/dataclass/test_pointcloud.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index e354bb06..a6464978 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -10,6 +10,7 @@ from attrs import define, field from t4_devkit.common.io import load_json +from t4_devkit.common.t4pack import PackReader, find_pack if TYPE_CHECKING: from typing_extensions import Self @@ -28,6 +29,27 @@ ] +def _load_pcd_bin(filepath: str, num_pts_feats: int) -> NDArrayFloat: + """Load points from a `.pcd.bin` file. + + If the file does not exist but the pack of its directory does, e.g. `data/LIDAR_CONCAT.pack`, + the frame is read from the pack instead. + + Args: + filepath (str): Path to the `.pcd.bin` file. + num_pts_feats (int): Number of float32 fields in each point. + + Returns: + NDArrayFloat: Points in the shape of (N, num_pts_feats). + """ + if not osp.exists(filepath) and (pack_filepath := find_pack(filepath)) is not None: + with PackReader(pack_filepath) as reader: + return reader.read_frame_by_name(osp.basename(filepath)) + + scan = np.fromfile(filepath, dtype=np.float32) + return scan.reshape((-1, num_pts_feats)) + + @define class Stamp: """A dataclass to represent timestamp. @@ -272,8 +294,7 @@ def from_file(cls, filepath: str, metainfo_filepath: str | None = None) -> Self: if filepath.endswith(".bin"): num_pts_feats = getattr(metainfo, "num_pts_feats", 5) - scan = np.fromfile(filepath, dtype=np.float32) - points = scan.reshape((-1, num_pts_feats))[:, : cls.num_dims()] + points = _load_pcd_bin(filepath, num_pts_feats)[:, : cls.num_dims()] else: pcd = pypcd4.PointCloud.from_path(filepath) points = pcd.numpy(("x", "y", "z", "intensity")).astype(np.float32) @@ -424,8 +445,7 @@ def from_file( if point_filepath.endswith(".bin"): num_pts_feats = getattr(metainfo, "num_pts_feats", 5) - scan = np.fromfile(point_filepath, dtype=np.float32) - points = scan.reshape((-1, num_pts_feats))[:, : cls.num_dims()] + points = _load_pcd_bin(point_filepath, num_pts_feats)[:, : cls.num_dims()] else: pcd = pypcd4.PointCloud.from_path(point_filepath) points = pcd.numpy(("x", "y", "z", "intensity")).astype(np.float32) diff --git a/tests/dataclass/test_pointcloud.py b/tests/dataclass/test_pointcloud.py index 411a5785..2af11025 100644 --- a/tests/dataclass/test_pointcloud.py +++ b/tests/dataclass/test_pointcloud.py @@ -6,6 +6,7 @@ import numpy as np import pytest +from t4_devkit.common.t4pack import pack_directory from t4_devkit.dataclass.pointcloud import ( LidarPointCloud, PointCloudMetainfo, @@ -418,3 +419,46 @@ def test_segmentation_pointcloud_from_file_reads_points_labels_and_metainfo_from assert np.array_equal(pointcloud.labels, labels) assert pointcloud.metainfo is not None assert pointcloud.metainfo.source_tokens == ["lidar_front"] + + +def test_lidar_pointcloud_from_file_reads_points_from_pack(tmp_path: Path) -> None: + lidar_dir = tmp_path / "data" / "LIDAR_CONCAT" + lidar_dir.mkdir(parents=True) + scans = [np.full((3, 5), i, dtype=np.float32) for i in range(3)] + for i, scan in enumerate(scans): + scan.tofile(lidar_dir / f"{i:05d}.pcd.bin") + pack_directory(lidar_dir, level=1) + + # remove a loose frame, then it is read from the pack + bin_filepath = lidar_dir / "00001.pcd.bin" + bin_filepath.unlink() + + pointcloud = LidarPointCloud.from_file(str(bin_filepath)) + + assert np.array_equal(pointcloud.points, scans[1][:, :4].T) + + +def test_lidar_pointcloud_from_file_raises_without_file_and_pack(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + LidarPointCloud.from_file(str(tmp_path / "LIDAR_CONCAT" / "00000.pcd.bin")) + + +def test_segmentation_pointcloud_from_file_reads_points_from_pack(tmp_path: Path) -> None: + lidar_dir = tmp_path / "data" / "LIDAR_CONCAT" + lidar_dir.mkdir(parents=True) + scan = np.arange(10, dtype=np.float32).reshape(2, 5) + scan.tofile(lidar_dir / "00000.pcd.bin") + pack_directory(lidar_dir, level=1) + (lidar_dir / "00000.pcd.bin").unlink() + + label_filepath = tmp_path / "labels.bin" + labels = np.array([1, 2], dtype=np.uint8) + labels.tofile(label_filepath) + + pointcloud = SegmentationPointCloud.from_file( + point_filepath=str(lidar_dir / "00000.pcd.bin"), + label_filepath=str(label_filepath), + ) + + assert np.array_equal(pointcloud.points, scan[:, :4].T) + assert np.array_equal(pointcloud.labels, labels) From 06cddb3eff3c13b3f2720d2af6bd0c65b576cddc Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 15:24:47 +0900 Subject: [PATCH 03/12] feat(cli): add t4pack command to pack and verify pointclouds Co-Authored-By: Claude Opus 5.5 --- t4_devkit/cli/pack.py | 129 ++++++++++++++++++++++++++++++++++++++ tests/cli/test_pack.py | 61 ++++++++++++++++++ tests/cli/test_version.py | 2 + 3 files changed, 192 insertions(+) create mode 100644 t4_devkit/cli/pack.py create mode 100644 tests/cli/test_pack.py diff --git a/t4_devkit/cli/pack.py b/t4_devkit/cli/pack.py new file mode 100644 index 00000000..2dd34f10 --- /dev/null +++ b/t4_devkit/cli/pack.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import os +import os.path as osp +import sys +from glob import glob +from typing import Annotated + +import typer +from rich.console import Console + +from t4_devkit.common.t4pack import FRAME_SUFFIX, pack_directory, verify_pack +from t4_devkit.dataclass.pointcloud import PointCloudMetainfo +from t4_devkit.tier4 import load_metadata + +from .version import version_callback + +console = Console() + +cli = typer.Typer( + name="t4pack", + no_args_is_help=True, + context_settings={"help_option_names": ["-h", "--help"]}, + pretty_exceptions_enable=False, +) + + +@cli.callback() +def main( + version: Annotated[ + bool, + typer.Option( + "--version", + "-v", + help="Show the application version and exit.", + callback=version_callback, + is_eager=True, + ), + ] = False, +) -> None: + pass + + +@cli.command("pack", help="Pack .pcd.bin frames of each channel into '.pack'.") +def pack( + data_root: Annotated[str, typer.Argument(help="Root directory path to the dataset.")], + revision: Annotated[ + str | None, + typer.Option( + ..., "-rv", "--revision", help="Specify if you want to pack the specific version." + ), + ] = None, + level: Annotated[int, typer.Option(..., "-l", "--level", help="zstd compression level.")] = 19, + overwrite: Annotated[ + bool, typer.Option(..., "--overwrite", help="Overwrite existing packs.") + ] = False, +) -> None: + channel_dirs = _find_channel_dirs(data_root, revision) + + for channel_dir in channel_dirs: + stats = pack_directory( + channel_dir, + num_pts_feats=_num_pts_feats(channel_dir), + level=level, + overwrite=overwrite, + ) + if stats is None: + console.print(f"[yellow]SKIP[/yellow] {channel_dir}: already packed") + else: + console.print( + f"[green]PACKED[/green] {channel_dir}: {stats.num_frames} frames, " + f"{stats.raw_bytes} -> {stats.packed_bytes} bytes ({stats.ratio:.2f}x)" + ) + + +@cli.command("verify", help="Verify that each '.pack' matches its .pcd.bin frames.") +def verify( + data_root: Annotated[str, typer.Argument(help="Root directory path to the dataset.")], + revision: Annotated[ + str | None, + typer.Option( + ..., "-rv", "--revision", help="Specify if you want to verify the specific version." + ), + ] = None, +) -> None: + channel_dirs = [ + channel_dir + for channel_dir in _find_channel_dirs(data_root, revision) + if osp.exists(channel_dir + ".pack") + ] + if not channel_dirs: + console.print("[red]No packs to verify[/red]") + sys.exit(1) + + is_passed = True + for channel_dir in channel_dirs: + mismatches = verify_pack(channel_dir, num_pts_feats=_num_pts_feats(channel_dir)) + if mismatches: + is_passed = False + console.print( + f"[red]FAILED[/red] {channel_dir}: {len(mismatches)} frames differ, " + f"e.g. {mismatches[:3]}" + ) + else: + console.print(f"[green]VERIFIED[/green] {channel_dir}") + + sys.exit(0 if is_passed else 1) + + +def _find_channel_dirs(data_root: str, revision: str | None) -> list[str]: + """Return directories under `data/` that contain .pcd.bin frames.""" + data_dir = osp.join(load_metadata(data_root, revision).data_root, "data") + if not osp.isdir(data_dir): + raise FileNotFoundError(f"Data directory is not found: {data_dir}") + + return sorted( + osp.join(data_dir, name) + for name in os.listdir(data_dir) + if osp.isdir(osp.join(data_dir, name)) + and glob(osp.join(data_dir, name, f"*{FRAME_SUFFIX}")) + ) + + +def _num_pts_feats(channel_dir: str) -> int: + """Return `num_pts_feats` from the metainfo in `_INFO/`, or 5 if there is none.""" + info_filepaths = sorted(glob(osp.join(f"{channel_dir}_INFO", "*.json"))) + if not info_filepaths: + return 5 + return PointCloudMetainfo.from_file(info_filepaths[0]).num_pts_feats diff --git a/tests/cli/test_pack.py b/tests/cli/test_pack.py new file mode 100644 index 00000000..076a3a7a --- /dev/null +++ b/tests/cli/test_pack.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +from typer.testing import CliRunner + +from t4_devkit.cli.pack import cli + + +def _create_dataset(tmp_path: Path) -> Path: + data_root = tmp_path / "dataset_id" + lidar_dir = data_root / "0" / "data" / "LIDAR_CONCAT" + lidar_dir.mkdir(parents=True) + (data_root / "0" / "data" / "CAM_FRONT").mkdir() + for i in range(3): + frame = np.zeros((10, 5), dtype=np.float32) + frame[:, :3] = i + frame.tofile(lidar_dir / f"{i:05d}.pcd.bin") + return data_root + + +def test_pack_and_verify(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + runner = CliRunner() + + result = runner.invoke(cli, ["pack", str(data_root), "--level", "1"]) + assert result.exit_code == 0, result.output + assert "PACKED" in result.output + assert (data_root / "0" / "data" / "LIDAR_CONCAT.pack").exists() + assert not (data_root / "0" / "data" / "CAM_FRONT.pack").exists() + + result = runner.invoke(cli, ["pack", str(data_root)]) + assert result.exit_code == 0, result.output + assert "SKIP" in result.output + + result = runner.invoke(cli, ["verify", str(data_root)]) + assert result.exit_code == 0, result.output + assert "VERIFIED" in result.output + + +def test_verify_fails_on_changed_frame(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + runner = CliRunner() + runner.invoke(cli, ["pack", str(data_root), "--level", "1"]) + + np.ones((10, 5), dtype=np.float32).tofile( + data_root / "0" / "data" / "LIDAR_CONCAT" / "00001.pcd.bin" + ) + + result = runner.invoke(cli, ["verify", str(data_root)]) + assert result.exit_code == 1 + assert "FAILED" in result.output + + +def test_verify_fails_without_pack(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + + result = CliRunner().invoke(cli, ["verify", str(data_root)]) + + assert result.exit_code == 1 diff --git a/tests/cli/test_version.py b/tests/cli/test_version.py index 7e671702..6bd3601a 100644 --- a/tests/cli/test_version.py +++ b/tests/cli/test_version.py @@ -6,6 +6,7 @@ from typer import Typer from typer.testing import CliRunner +from t4_devkit.cli.pack import cli as pack_cli from t4_devkit.cli.sanity import cli as sanity_cli from t4_devkit.cli.visualize import cli as visualize_cli @@ -14,6 +15,7 @@ ("cli", "name"), [ (sanity_cli, "t4sanity"), + (pack_cli, "t4pack"), (visualize_cli, "t4viz"), ], ) From 7b4dbd6e8b1636e67a57ea338f48a9e7832d060f Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 15:24:47 +0900 Subject: [PATCH 04/12] docs: describe packed pointcloud and t4pack CLI Co-Authored-By: Claude Opus 5.5 --- docs/cli/index.md | 1 + docs/cli/t4pack.md | 79 +++++++++++++++++++++++++++++++++++++++++++++ docs/schema/data.md | 36 +++++++++++++++++++++ mkdocs.yaml | 1 + 4 files changed, 117 insertions(+) create mode 100644 docs/cli/t4pack.md diff --git a/docs/cli/index.md b/docs/cli/index.md index 7f7d48d6..1c762d43 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -4,3 +4,4 @@ Following command line tools are supported: - [`t4viz`: Visualize T4 dataset features](./t4viz.md) - [`t4sanity`: Sanity checker of T4 dataset](./t4sanity.md) +- [`t4pack`: Pack pointcloud frames into a single compressed file](./t4pack.md) diff --git a/docs/cli/t4pack.md b/docs/cli/t4pack.md new file mode 100644 index 00000000..4414368a --- /dev/null +++ b/docs/cli/t4pack.md @@ -0,0 +1,79 @@ +`t4pack` packs the `.pcd.bin` pointcloud frames of each sensor channel into a single compressed file, `data/.pack`. +See [Packed Pointcloud](../schema/data.md#packed-pointcloud) for the format. + +```shell +$ t4pack -h + + Usage: t4pack [OPTIONS] COMMAND [ARGS]... + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --version -v Show the application version and exit. │ +│ --install-completion Install completion for the current shell. │ +│ --show-completion Show completion for the current shell, to copy it or customize the installation. │ +│ --help -h Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ pack Pack .pcd.bin frames of each channel into '.pack'. │ +│ verify Verify that each '.pack' matches its .pcd.bin frames. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +## Shell Completion + +Run the following command to install completion, and reload shell. + +```{ .shell .copy } +t4pack --install-completion +``` + +## Usages + +### Pack Pointclouds + +```shell +$ t4pack pack -h + + Usage: t4pack pack [OPTIONS] DATA_ROOT + + Pack .pcd.bin frames of each channel into '.pack'. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ * data_root TEXT Root directory path to the dataset. [required] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --revision -rv TEXT Specify if you want to pack the specific version. │ +│ --level -l INTEGER zstd compression level. [default: 19] │ +│ --overwrite Overwrite existing packs. │ +│ --help -h Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +For example, the following command packs `data/LIDAR_CONCAT/*.pcd.bin` into `data/LIDAR_CONCAT.pack`: + +```shell +t4pack pack +``` + +Every frame is verified bit-exact against its source before the pack is put in place, so an existing pack is always complete. +The source `.pcd.bin` files are kept. +A channel that is already packed is skipped unless `--overwrite` is specified. + +### Verify Packs + +```shell +$ t4pack verify -h + + Usage: t4pack verify [OPTIONS] DATA_ROOT + + Verify that each '.pack' matches its .pcd.bin frames. + +╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ * data_root TEXT Root directory path to the dataset. [required] │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --revision -rv TEXT Specify if you want to verify the specific version. │ +│ --help -h Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +The command exits with `1` if any frame is missing from the pack or differs from its source. diff --git a/docs/schema/data.md b/docs/schema/data.md index 09bb6751..c53a6522 100644 --- a/docs/schema/data.md +++ b/docs/schema/data.md @@ -77,6 +77,42 @@ Example pointcloud metainfo JSON: } ``` +### Packed Pointcloud + +The `.pcd.bin` frames of a channel can be packed into a single file next to the directory, `data/.pack`, with [`t4pack`](../cli/t4pack.md): + +```shell +data/ +├── LIDAR_CONCAT +│ ├── .pcd.bin +│ ... +└── LIDAR_CONCAT.pack +``` + +When `.pcd.bin` does not exist but `.pack` does, `LidarPointCloud.from_file()` and `SegmentationPointCloud.from_file()` read the frame from the pack by its file name, so `sample_data.filename` does not need to change. + +The pack (t4pack v1) stores each frame as an independently compressed zstd blob, followed by a JSON index of the frames: + +```shell +magic b'T4PACK\x00\x01' +blob_0 .. blob_{N-1} zstd(column_0 bytes | column_1 bytes | ...) +zstd(JSON index) name, offset, size, n_points, dtypes per frame + little-endian +magic +``` + +Points are stored column-major. `x, y, z` (and other float features) are byte-shuffled `float32`, so they round-trip bit-exact, while `intensity` and `ring` are stored as `uint8` and `int8`. +The column layout is chosen by `num_pts_feats` in the pointcloud metainfo JSON. + +Each frame can also be loaded directly as follows: + +```python +from t4_devkit.common.t4pack import PackReader + +with PackReader("data/LIDAR_CONCAT.pack") as reader: + points = reader.read_frame_by_name("00000.pcd.bin") # (N, num_pts_feats) float32 +``` + ## Camera Image Camera directory contains raw images as the name of ``: diff --git a/mkdocs.yaml b/mkdocs.yaml index 942f1899..2b2db7d2 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -20,6 +20,7 @@ nav: - Home: cli/index.md - t4viz: cli/t4viz.md - t4sanity: cli/t4sanity.md + - t4pack: cli/t4pack.md - API References: - t4_devkit.tier4: apis/tier4.md - t4_devkit.helper: apis/helper.md From e14bf95dd1b96a25cedfbc4fed10221f3a70e77e Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 16:03:47 +0900 Subject: [PATCH 05/12] feat(common): delete loose frames after verifying pack and fall back to lossless layout Co-Authored-By: Claude Opus 5.5 --- t4_devkit/common/t4pack.py | 90 +++++++++++++++++++++++++++++------- tests/common/test_t4pack.py | 91 +++++++++++++++++++++++++++++++++++-- 2 files changed, 161 insertions(+), 20 deletions(-) diff --git a/t4_devkit/common/t4pack.py b/t4_devkit/common/t4pack.py index 541b614c..ded26dc4 100644 --- a/t4_devkit/common/t4pack.py +++ b/t4_devkit/common/t4pack.py @@ -45,6 +45,7 @@ "write_pack", "pack_directory", "verify_pack", + "delete_loose_frames", "find_pack", ) @@ -103,6 +104,14 @@ def from_num_pts_feats(cls, num_pts_feats: int) -> Self: else: raise ValueError(f"Expected num_pts_feats >= 4, but got {num_pts_feats}") + def to_lossless(self) -> Self: + """Return the layout that stores every column as byte-shuffled float32. + + Returns: + Self: Lossless layout with the same fields. + """ + return self.__class__(dtypes=("f4s",) * len(self.dtypes), fields=self.fields) + @define class PackStats: @@ -390,7 +399,9 @@ def pack_directory( """Pack all `.pcd.bin` frames in a directory into `.pack`. Every frame is verified bit-exact against its source before the pack is put in place, - so a pack that exists is always complete. Source frames are not deleted. + so a pack that exists is always complete. If the 1-byte columns of the layout cannot hold + the source values, e.g. non-integer intensity, every column is stored as float32 instead. Source frames are not deleted here, + use `delete_loose_frames` to delete them. Args: directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. @@ -421,15 +432,18 @@ def pack_directory( # write to a temporary file, verify it, and then rename it to the final path tmp_filepath = pack_filepath + ".unverified" try: - num_frames = write_pack( - tmp_filepath, - (_load_frame(filepath, num_pts_feats) for filepath in filepaths), - layout=layout, - names=names, - level=level, - ) - mismatches = _compare_frames(tmp_filepath, directory, names, num_pts_feats) - if mismatches: + for candidate in dict.fromkeys((layout, layout.to_lossless())): + num_frames = write_pack( + tmp_filepath, + (_load_frame(filepath, num_pts_feats) for filepath in filepaths), + layout=candidate, + names=names, + level=level, + ) + mismatches = _compare_frames(tmp_filepath, directory, names, num_pts_feats) + if not mismatches: + break + else: raise ValueError( f"{len(mismatches)} frames differ after packing {directory}, e.g. {mismatches[:3]}" ) @@ -446,19 +460,22 @@ def pack_directory( def verify_pack(directory: PathLike, num_pts_feats: int = 5) -> list[str]: - """Verify that every frame in `.pack` equals its source `.pcd.bin` frame. + """Verify `.pack` against the `.pcd.bin` frames in the directory. + + A frame whose source exists must equal it bit-exact, a source frame must be in the pack, + and a frame whose source has been deleted must be decodable. Args: directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. num_pts_feats (int, optional): Number of float32 fields in each point. Returns: - list[str]: Names of the frames that are missing or differ. Empty if the pack is valid. + list[str]: Names of the frames that are missing, differ or are broken. + Empty if the pack is valid. """ directory = os.fspath(directory).rstrip(os.sep) - return _compare_frames( - directory + PACK_SUFFIX, directory, _list_frames(directory), num_pts_feats - ) + names = _list_frames(directory) if osp.isdir(directory) else [] + return _compare_frames(directory + PACK_SUFFIX, directory, names, num_pts_feats) def _compare_frames( @@ -470,7 +487,15 @@ def _compare_frames( mismatches = [] with PackReader(pack_filepath) as reader: source_names = set(names) - mismatches.extend(name for name in reader.names if name not in source_names) + for index, name in enumerate(reader.names): + if name in source_names: + continue + # the source frame has been deleted, so only check that the frame can be decoded + try: + reader.read_frame(index) + except Exception: + mismatches.append(name) + for name in names: try: packed = reader.read_frame_by_name(name) @@ -482,6 +507,39 @@ def _compare_frames( return mismatches +def delete_loose_frames(directory: PathLike, num_pts_feats: int = 5) -> int: + """Delete the `.pcd.bin` frames in a directory after verifying `.pack`. + + Nothing is deleted unless every frame in the directory equals its frame in the pack. + The directory itself is removed if it becomes empty. + + Args: + directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. + num_pts_feats (int, optional): Number of float32 fields in each point. + + Returns: + int: Number of deleted frames. + """ + directory = os.fspath(directory).rstrip(os.sep) + if not osp.exists(directory + PACK_SUFFIX): + raise FileNotFoundError(f"Pack is not found: {directory + PACK_SUFFIX}") + + mismatches = verify_pack(directory, num_pts_feats=num_pts_feats) + if mismatches: + raise ValueError( + f"{len(mismatches)} frames differ from the pack of {directory}, e.g. {mismatches[:3]}" + ) + + names = _list_frames(directory) if osp.isdir(directory) else [] + for name in names: + os.remove(osp.join(directory, name)) + + if osp.isdir(directory) and not os.listdir(directory): + os.rmdir(directory) + + return len(names) + + def find_pack(filepath: PathLike) -> str | None: """Return the path to the pack containing the specified frame if it exists. diff --git a/tests/common/test_t4pack.py b/tests/common/test_t4pack.py index b0e33152..e9311cf4 100644 --- a/tests/common/test_t4pack.py +++ b/tests/common/test_t4pack.py @@ -9,6 +9,7 @@ PackLayout, PackReader, decode_frame, + delete_loose_frames, encode_frame, find_pack, pack_directory, @@ -158,15 +159,26 @@ def test_pack_directory_supports_wide_points(tmp_path: Path) -> None: assert verify_pack(directory, num_pts_feats=7) == [] -def test_pack_directory_refuses_lossy_pack(tmp_path: Path) -> None: +def test_pack_directory_falls_back_to_lossless_layout(tmp_path: Path) -> None: directory = tmp_path / "LIDAR_CONCAT" frame = _frame(100) frame[0, 3] = 0.5 # intensity is not an integer _write_frames(directory, [frame], ["00000.pcd.bin"]) - with pytest.raises(ValueError): - pack_directory(directory, level=1) - assert list(tmp_path.glob("*.pack*")) == [] + pack_directory(directory, level=1) + + assert sorted(p.name for p in tmp_path.glob("*.pack*")) == ["LIDAR_CONCAT.pack"] + with PackReader(tmp_path / "LIDAR_CONCAT.pack") as reader: + assert reader.frames[0]["dtypes"] == ["f4s"] * 5 + assert np.array_equal(reader.read_frame(0), frame) + assert verify_pack(directory) == [] + + +def test_pack_layout_to_lossless() -> None: + layout = PackLayout.from_num_pts_feats(5).to_lossless() + + assert layout.dtypes == ("f4s",) * 5 + assert layout.fields == ("x", "y", "z", "intensity", "ring") def test_pack_directory_requires_frames(tmp_path: Path) -> None: @@ -177,6 +189,77 @@ def test_pack_directory_requires_frames(tmp_path: Path) -> None: pack_directory(directory) +def test_verify_pack_after_deleting_loose_frames(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + names = ["00000.pcd.bin", "00001.pcd.bin"] + _write_frames(directory, [_frame(100, seed=i) for i in range(len(names))], names) + pack_directory(directory, level=1) + + # partially deleted + (directory / names[0]).unlink() + assert verify_pack(directory) == [] + + # fully deleted + (directory / names[1]).unlink() + directory.rmdir() + assert verify_pack(directory) == [] + + +def test_delete_loose_frames(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + names = ["00000.pcd.bin", "00001.pcd.bin"] + frames = [_frame(100, seed=i) for i in range(len(names))] + _write_frames(directory, frames, names) + pack_directory(directory, level=1) + + assert delete_loose_frames(directory) == 2 + assert not directory.exists() + with PackReader(tmp_path / "LIDAR_CONCAT.pack") as reader: + assert np.array_equal(reader.read_frame_by_name(names[1]), frames[1]) + + # nothing left to delete + assert delete_loose_frames(directory) == 0 + + +def test_delete_loose_frames_keeps_other_files(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + _write_frames(directory, [_frame(100)], ["00000.pcd.bin"]) + (directory / "README.txt").write_text("keep me") + pack_directory(directory, level=1) + + assert delete_loose_frames(directory) == 1 + assert sorted(p.name for p in directory.iterdir()) == ["README.txt"] + + +def test_delete_loose_frames_refuses_mismatched_pack(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + names = ["00000.pcd.bin", "00001.pcd.bin"] + _write_frames(directory, [_frame(100, seed=i) for i in range(len(names))], names) + pack_directory(directory, level=1) + + # changed frame + _frame(100, seed=99).tofile(directory / names[1]) + with pytest.raises(ValueError): + delete_loose_frames(directory) + + # frame that is not in the pack + pack_directory(directory, level=1, overwrite=True) + _frame(100, seed=2).tofile(directory / "00002.pcd.bin") + with pytest.raises(ValueError): + delete_loose_frames(directory) + + assert sorted(p.name for p in directory.iterdir()) == [*names, "00002.pcd.bin"] + + +def test_delete_loose_frames_requires_pack(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + _write_frames(directory, [_frame(100)], ["00000.pcd.bin"]) + + with pytest.raises(FileNotFoundError): + delete_loose_frames(directory) + assert (directory / "00000.pcd.bin").exists() + + def test_find_pack(tmp_path: Path) -> None: filepath = tmp_path / "LIDAR_CONCAT" / "00000.pcd.bin" From 8ab0da29ec2f1dceaee2c9ddb9bc835c76657404 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 16:03:47 +0900 Subject: [PATCH 06/12] feat(cli): add --delete-loose option to t4pack pack Co-Authored-By: Claude Opus 5.5 --- docs/cli/t4pack.md | 26 ++++++++++++++------- t4_devkit/cli/pack.py | 52 ++++++++++++++++++++++++++++++++++-------- tests/cli/test_pack.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 18 deletions(-) diff --git a/docs/cli/t4pack.md b/docs/cli/t4pack.md index 4414368a..25a77e0d 100644 --- a/docs/cli/t4pack.md +++ b/docs/cli/t4pack.md @@ -14,7 +14,7 @@ $ t4pack -h ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ pack Pack .pcd.bin frames of each channel into '.pack'. │ -│ verify Verify that each '.pack' matches its .pcd.bin frames. │ +│ verify Verify that each '.pack' matches its .pcd.bin frames if they exist. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -41,10 +41,11 @@ $ t4pack pack -h │ * data_root TEXT Root directory path to the dataset. [required] │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ --revision -rv TEXT Specify if you want to pack the specific version. │ -│ --level -l INTEGER zstd compression level. [default: 19] │ -│ --overwrite Overwrite existing packs. │ -│ --help -h Show this message and exit. │ +│ --revision -rv TEXT Specify if you want to pack the specific version. │ +│ --level -l INTEGER zstd compression level. [default: 19] │ +│ --overwrite Overwrite existing packs. │ +│ --delete-loose Delete .pcd.bin frames after verifying them against the pack. │ +│ --help -h Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` @@ -55,9 +56,17 @@ t4pack pack ``` Every frame is verified bit-exact against its source before the pack is put in place, so an existing pack is always complete. -The source `.pcd.bin` files are kept. +If the source values do not fit the compact `uint8`/`int8` columns, e.g. non-integer intensity, every column is stored as `float32` instead. A channel that is already packed is skipped unless `--overwrite` is specified. +The source `.pcd.bin` files are kept by default. With `--delete-loose`, they are deleted after every frame is verified against the pack again, which also applies to a channel that is already packed: + +```shell +t4pack pack --delete-loose +``` + +Nothing is deleted from a channel if any of its frames is missing from the pack or differs from it, and files other than `.pcd.bin` are kept. + ### Verify Packs ```shell @@ -65,7 +74,7 @@ $ t4pack verify -h Usage: t4pack verify [OPTIONS] DATA_ROOT - Verify that each '.pack' matches its .pcd.bin frames. + Verify that each '.pack' matches its .pcd.bin frames if they exist. ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ * data_root TEXT Root directory path to the dataset. [required] │ @@ -76,4 +85,5 @@ $ t4pack verify -h ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` -The command exits with `1` if any frame is missing from the pack or differs from its source. +Each frame whose `.pcd.bin` still exists is compared bit-exact with the pack, and each frame whose `.pcd.bin` has been deleted is checked to be decodable. +The command exits with `1` if any frame is missing from the pack, differs from its source or is broken. diff --git a/t4_devkit/cli/pack.py b/t4_devkit/cli/pack.py index 2dd34f10..5f3e5073 100644 --- a/t4_devkit/cli/pack.py +++ b/t4_devkit/cli/pack.py @@ -9,7 +9,13 @@ import typer from rich.console import Console -from t4_devkit.common.t4pack import FRAME_SUFFIX, pack_directory, verify_pack +from t4_devkit.common.t4pack import ( + FRAME_SUFFIX, + PACK_SUFFIX, + delete_loose_frames, + pack_directory, + verify_pack, +) from t4_devkit.dataclass.pointcloud import PointCloudMetainfo from t4_devkit.tier4 import load_metadata @@ -54,13 +60,22 @@ def pack( overwrite: Annotated[ bool, typer.Option(..., "--overwrite", help="Overwrite existing packs.") ] = False, + delete_loose: Annotated[ + bool, + typer.Option( + ..., + "--delete-loose", + help="Delete .pcd.bin frames after verifying them against the pack.", + ), + ] = False, ) -> None: channel_dirs = _find_channel_dirs(data_root, revision) for channel_dir in channel_dirs: + num_pts_feats = _num_pts_feats(channel_dir) stats = pack_directory( channel_dir, - num_pts_feats=_num_pts_feats(channel_dir), + num_pts_feats=num_pts_feats, level=level, overwrite=overwrite, ) @@ -72,8 +87,14 @@ def pack( f"{stats.raw_bytes} -> {stats.packed_bytes} bytes ({stats.ratio:.2f}x)" ) + if delete_loose: + num_deleted = delete_loose_frames(channel_dir, num_pts_feats=num_pts_feats) + console.print(f"[green]DELETED[/green] {channel_dir}: {num_deleted} frames") -@cli.command("verify", help="Verify that each '.pack' matches its .pcd.bin frames.") + +@cli.command( + "verify", help="Verify that each '.pack' matches its .pcd.bin frames if they exist." +) def verify( data_root: Annotated[str, typer.Argument(help="Root directory path to the dataset.")], revision: Annotated[ @@ -83,11 +104,7 @@ def verify( ), ] = None, ) -> None: - channel_dirs = [ - channel_dir - for channel_dir in _find_channel_dirs(data_root, revision) - if osp.exists(channel_dir + ".pack") - ] + channel_dirs = _find_packed_channel_dirs(data_root, revision) if not channel_dirs: console.print("[red]No packs to verify[/red]") sys.exit(1) @@ -107,12 +124,17 @@ def verify( sys.exit(0 if is_passed else 1) -def _find_channel_dirs(data_root: str, revision: str | None) -> list[str]: - """Return directories under `data/` that contain .pcd.bin frames.""" +def _find_data_dir(data_root: str, revision: str | None) -> str: + """Return the path to `data/` of the dataset.""" data_dir = osp.join(load_metadata(data_root, revision).data_root, "data") if not osp.isdir(data_dir): raise FileNotFoundError(f"Data directory is not found: {data_dir}") + return data_dir + +def _find_channel_dirs(data_root: str, revision: str | None) -> list[str]: + """Return directories under `data/` that contain .pcd.bin frames.""" + data_dir = _find_data_dir(data_root, revision) return sorted( osp.join(data_dir, name) for name in os.listdir(data_dir) @@ -121,6 +143,16 @@ def _find_channel_dirs(data_root: str, revision: str | None) -> list[str]: ) +def _find_packed_channel_dirs(data_root: str, revision: str | None) -> list[str]: + """Return channel directories under `data/` that have a pack, even if they are deleted.""" + data_dir = _find_data_dir(data_root, revision) + return sorted( + osp.join(data_dir, name.removesuffix(PACK_SUFFIX)) + for name in os.listdir(data_dir) + if name.endswith(PACK_SUFFIX) + ) + + def _num_pts_feats(channel_dir: str) -> int: """Return `num_pts_feats` from the metainfo in `_INFO/`, or 5 if there is none.""" info_filepaths = sorted(glob(osp.join(f"{channel_dir}_INFO", "*.json"))) diff --git a/tests/cli/test_pack.py b/tests/cli/test_pack.py index 076a3a7a..fb106611 100644 --- a/tests/cli/test_pack.py +++ b/tests/cli/test_pack.py @@ -39,6 +39,47 @@ def test_pack_and_verify(tmp_path: Path) -> None: assert "VERIFIED" in result.output +def test_pack_and_delete_loose(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + lidar_dir = data_root / "0" / "data" / "LIDAR_CONCAT" + runner = CliRunner() + + result = runner.invoke(cli, ["pack", str(data_root), "--level", "1", "--delete-loose"]) + assert result.exit_code == 0, result.output + assert "DELETED" in result.output + assert not lidar_dir.exists() + + result = runner.invoke(cli, ["verify", str(data_root)]) + assert result.exit_code == 0, result.output + assert "VERIFIED" in result.output + + +def test_delete_loose_on_already_packed(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + lidar_dir = data_root / "0" / "data" / "LIDAR_CONCAT" + runner = CliRunner() + runner.invoke(cli, ["pack", str(data_root), "--level", "1"]) + assert lidar_dir.exists() + + result = runner.invoke(cli, ["pack", str(data_root), "--delete-loose"]) + assert result.exit_code == 0, result.output + assert "SKIP" in result.output + assert "DELETED" in result.output + assert not lidar_dir.exists() + + +def test_delete_loose_refuses_changed_frame(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + lidar_dir = data_root / "0" / "data" / "LIDAR_CONCAT" + runner = CliRunner() + runner.invoke(cli, ["pack", str(data_root), "--level", "1"]) + np.ones((10, 5), dtype=np.float32).tofile(lidar_dir / "00001.pcd.bin") + + result = runner.invoke(cli, ["pack", str(data_root), "--delete-loose"]) + assert result.exit_code != 0 + assert len(list(lidar_dir.glob("*.pcd.bin"))) == 3 + + def test_verify_fails_on_changed_frame(tmp_path: Path) -> None: data_root = _create_dataset(tmp_path) runner = CliRunner() From 5a363e3f526de6874a711dcbfca5b8d89f7fb284 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 16:03:47 +0900 Subject: [PATCH 07/12] feat(sanity): accept pcd.bin frames stored in pack for REF201 Co-Authored-By: Claude Opus 5.5 --- docs/schema/data.md | 1 + docs/schema/requirement.md | 2 +- t4_devkit/sanity/reference/ref201.py | 35 +++++++++++++++++++++++-- tests/sanity/test_reference_checkers.py | 29 ++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/schema/data.md b/docs/schema/data.md index c53a6522..d9992842 100644 --- a/docs/schema/data.md +++ b/docs/schema/data.md @@ -90,6 +90,7 @@ data/ ``` When `.pcd.bin` does not exist but `.pack` does, `LidarPointCloud.from_file()` and `SegmentationPointCloud.from_file()` read the frame from the pack by its file name, so `sample_data.filename` does not need to change. +Therefore, the `.pcd.bin` files can be deleted after packing (`t4pack pack --delete-loose`), and the sanity check `REF201` accepts a `.pcd.bin` frame stored in its pack. The pack (t4pack v1) stores each frame as an independently compressed zstd blob, followed by a JSON index of the frames: diff --git a/docs/schema/requirement.md b/docs/schema/requirement.md index 8e5f8507..b0992f44 100644 --- a/docs/schema/requirement.md +++ b/docs/schema/requirement.md @@ -62,7 +62,7 @@ | ID | Name | Severity | Fixable | Description | | -------- | ------------------------------------ | -------- | ------- | ------------------------------------------------------ | -| `REF201` | `sample-data-filename-presence` | `ERROR` | `N/A` | `SampleData.filename` exists. | +| `REF201` | `sample-data-filename-presence` | `ERROR` | `N/A` | `SampleData.filename` exists as a file or in its pack. | | `REF202` | `sample-data-info-filename-presence` | `ERROR` | `N/A` | `SampleData.info_filename` exists if it is not `None`. | | `REF203` | `lidarseg-filename-presence` | `ERROR` | `N/A` | `LidarSeg.filename` exists if `lidarseg.json` exists. | diff --git a/t4_devkit/sanity/reference/ref201.py b/t4_devkit/sanity/reference/ref201.py index 168ebd86..bbeff941 100644 --- a/t4_devkit/sanity/reference/ref201.py +++ b/t4_devkit/sanity/reference/ref201.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from t4_devkit.common.t4pack import FRAME_SUFFIX, PackReader, find_pack from t4_devkit.schema import SchemaName from ..checker import RuleID, RuleName, Severity @@ -11,6 +12,8 @@ from .base import FileReferenceChecker if TYPE_CHECKING: + from pathlib import Path + from ..context import SanityContext @@ -24,15 +27,43 @@ class REF201(FileReferenceChecker): id = RuleID("REF201") name = RuleName("sample-data-filename-presence") severity = Severity.ERROR - description = "'SampleData.filename' exists." + description = "'SampleData.filename' exists as a file or in its pack." schema = SchemaName.SAMPLE_DATA def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() + packed_names: dict[str, set[str]] = {} return [ Reason(f"File not found: {record['filename']}") for record in records - if record.get("is_valid", True) and not data_root.joinpath(record["filename"]).exists() + if record.get("is_valid", True) + and not _exists(data_root.joinpath(record["filename"]), packed_names) ] or None + + +def _exists(filepath: Path, packed_names: dict[str, set[str]]) -> bool: + """Return whether the file exists, or is a `.pcd.bin` frame stored in its pack. + + Args: + filepath (Path): Path to the file. + packed_names (dict[str, set[str]]): Cache of frame names for each pack path. + + Returns: + bool: True if the file exists. + """ + if filepath.exists(): + return True + + if not filepath.name.endswith(FRAME_SUFFIX) or (pack_filepath := find_pack(filepath)) is None: + return False + + if pack_filepath not in packed_names: + try: + with PackReader(pack_filepath) as reader: + packed_names[pack_filepath] = set(reader.names) + except Exception: + packed_names[pack_filepath] = set() + + return filepath.name in packed_names[pack_filepath] diff --git a/tests/sanity/test_reference_checkers.py b/tests/sanity/test_reference_checkers.py index e5dd4704..c32081ed 100644 --- a/tests/sanity/test_reference_checkers.py +++ b/tests/sanity/test_reference_checkers.py @@ -6,6 +6,7 @@ import pytest +from t4_devkit.common.t4pack import delete_loose_frames, pack_directory from t4_devkit.sanity.context import SanityContext from t4_devkit.sanity.reference.ref001 import REF001 from t4_devkit.sanity.reference.ref002 import REF002 @@ -319,6 +320,34 @@ def test_ref201_fail_missing_filename(tmp_path: Path) -> None: assert any("does_not_exist.jpg" in r for r in report.reasons) + +def test_ref201_pass_packed_frames(tmp_path: Path) -> None: + """REF201 passes if loose .pcd.bin frames are deleted but stored in their pack.""" + root = _copy_dataset(tmp_path / "dataset_ref201_packed") + lidar_dir = root / "data" / "LIDAR_TOP" + pack_directory(lidar_dir, level=1) + delete_loose_frames(lidar_dir) + assert not lidar_dir.exists() + + checker = REF201() + report = checker(_context(root)) + assert report.is_passed(strict=True) + + +def test_ref201_fail_frame_missing_in_pack(tmp_path: Path) -> None: + """REF201 fails if a deleted .pcd.bin frame is not stored in its pack.""" + root = _copy_dataset(tmp_path / "dataset_ref201_packed_fail") + lidar_dir = root / "data" / "LIDAR_TOP" + (lidar_dir / "003.pcd.bin").rename(tmp_path / "003.pcd.bin") + pack_directory(lidar_dir, level=1) + delete_loose_frames(lidar_dir) + + checker = REF201() + report = checker(_context(root)) + assert not report.is_passed(strict=True) + assert list(report.reasons) == ["File not found: data/LIDAR_TOP/003.pcd.bin"] + + def test_ref202_fail_missing_info_filename(tmp_path: Path) -> None: """Add an info_filename pointing to a non-existing file to trigger REF202 failure.""" root = _copy_dataset(tmp_path / "dataset_ref014_fail") From c9effde3db42d0c015a2f84f5de7d40fd20a98f5 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Wed, 21 Oct 2026 16:23:03 +0900 Subject: [PATCH 08/12] fix(common): make packing and deleting loose frames safe for concurrent workers Co-Authored-By: Claude Opus 5.5 --- docs/cli/t4pack.md | 3 ++ t4_devkit/common/t4pack.py | 77 ++++++++++++++++++++++++++++--------- tests/common/test_t4pack.py | 56 +++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 18 deletions(-) diff --git a/docs/cli/t4pack.md b/docs/cli/t4pack.md index 25a77e0d..a3b9de31 100644 --- a/docs/cli/t4pack.md +++ b/docs/cli/t4pack.md @@ -67,6 +67,9 @@ t4pack pack --delete-loose Nothing is deleted from a channel if any of its frames is missing from the pack or differs from it, and files other than `.pcd.bin` are kept. +`t4pack pack` can be run by multiple workers at the same time, even on the same dataset: each worker writes and verifies its own temporary file, and a verified pack is put in place atomically. +`--overwrite` is refused if the existing pack contains frames whose `.pcd.bin` has already been deleted, so those frames are never lost. + ### Verify Packs ```shell diff --git a/t4_devkit/common/t4pack.py b/t4_devkit/common/t4pack.py index ded26dc4..194669bb 100644 --- a/t4_devkit/common/t4pack.py +++ b/t4_devkit/common/t4pack.py @@ -24,6 +24,7 @@ import os import os.path as osp import struct +import uuid from typing import TYPE_CHECKING, Iterable, Sequence import numpy as np @@ -207,8 +208,9 @@ def write_pack( ) -> int: """Write frames into a single pack file. - The pack is written to `.tmp` first and renamed, - so an interrupted write never leaves a partial pack in place. + The pack is written to a unique temporary file in the same directory first and renamed, + so an interrupted write never leaves a partial pack in place, + and concurrent writers never share a temporary file. Args: filepath (PathLike): Path to the output pack file. @@ -222,7 +224,7 @@ def write_pack( int: Number of written frames. """ filepath = os.fspath(filepath) - tmp_filepath = f"{filepath}.tmp" + tmp_filepath = _make_tmp_file(filepath) compressor = zstd.ZstdCompressor(level=level) entries = [] try: @@ -379,6 +381,18 @@ def read_frame_by_name(self, name: str) -> NDArrayF32: return self.read_frame(self._name_to_index[name]) +def _make_tmp_file(filepath: str) -> str: + """Create a unique temporary file next to the specified path and return its path. + + Unlike `tempfile.mkstemp`, the file is created with the default permissions under umask, + so the pack renamed from it is readable by other users as the source frames are. + """ + tmp_filepath = f"{filepath}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp" + with open(tmp_filepath, "xb"): + pass + return tmp_filepath + + def _load_frame(filepath: str, num_pts_feats: int) -> NDArrayF32: scan = np.fromfile(filepath, dtype=np.float32) if scan.size % num_pts_feats != 0: @@ -400,8 +414,11 @@ def pack_directory( Every frame is verified bit-exact against its source before the pack is put in place, so a pack that exists is always complete. If the 1-byte columns of the layout cannot hold - the source values, e.g. non-integer intensity, every column is stored as float32 instead. Source frames are not deleted here, - use `delete_loose_frames` to delete them. + the source values, e.g. non-integer intensity, every column is stored as float32 instead. + Source frames are not deleted here, use `delete_loose_frames` to delete them. + + It is safe to run concurrently on the same directory: each call writes and verifies + its own temporary file, and the last verified pack is put in place atomically. Args: directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. @@ -415,14 +432,25 @@ def pack_directory( directory = os.fspath(directory).rstrip(os.sep) pack_filepath = directory + PACK_SUFFIX - if osp.exists(pack_filepath) and not overwrite: + names = _list_frames(directory) if osp.isdir(directory) else [] + + if osp.exists(pack_filepath): try: - with PackReader(pack_filepath): - return None + with PackReader(pack_filepath) as reader: + packed_names = reader.names except Exception: - pass # broken pack, rewrite it + packed_names = None # broken pack, rewrite it + + if packed_names is not None: + if not overwrite: + return None + # frames whose source has been deleted would be lost by rewriting the pack + if dropped := sorted(set(packed_names) - set(names)): + raise ValueError( + f"Cannot overwrite {pack_filepath}, {len(dropped)} frames have no source, " + f"e.g. {dropped[:3]}" + ) - names = _list_frames(directory) if not names: raise FileNotFoundError(f"No {FRAME_SUFFIX} frames in {directory}") @@ -430,7 +458,7 @@ def pack_directory( layout = PackLayout.from_num_pts_feats(num_pts_feats) # write to a temporary file, verify it, and then rename it to the final path - tmp_filepath = pack_filepath + ".unverified" + tmp_filepath = _make_tmp_file(pack_filepath) try: for candidate in dict.fromkeys((layout, layout.to_lossless())): num_frames = write_pack( @@ -502,7 +530,11 @@ def _compare_frames( except KeyError: mismatches.append(name) continue - if not np.array_equal(packed, _load_frame(osp.join(directory, name), num_pts_feats)): + try: + source = _load_frame(osp.join(directory, name), num_pts_feats) + except FileNotFoundError: + continue # deleted by another process after listing, the frame is decodable + if not np.array_equal(packed, source): mismatches.append(name) return mismatches @@ -510,8 +542,10 @@ def _compare_frames( def delete_loose_frames(directory: PathLike, num_pts_feats: int = 5) -> int: """Delete the `.pcd.bin` frames in a directory after verifying `.pack`. - Nothing is deleted unless every frame in the directory equals its frame in the pack. + Nothing is deleted unless every frame in the directory equals its frame in the pack, + and only the frames that have been verified are deleted. The directory itself is removed if it becomes empty. + It is safe to run concurrently on the same directory. Args: directory (PathLike): Path to the directory of `.pcd.bin` frames, e.g. `data/LIDAR_CONCAT`. @@ -524,20 +558,27 @@ def delete_loose_frames(directory: PathLike, num_pts_feats: int = 5) -> int: if not osp.exists(directory + PACK_SUFFIX): raise FileNotFoundError(f"Pack is not found: {directory + PACK_SUFFIX}") - mismatches = verify_pack(directory, num_pts_feats=num_pts_feats) + names = _list_frames(directory) if osp.isdir(directory) else [] + mismatches = _compare_frames(directory + PACK_SUFFIX, directory, names, num_pts_feats) if mismatches: raise ValueError( f"{len(mismatches)} frames differ from the pack of {directory}, e.g. {mismatches[:3]}" ) - names = _list_frames(directory) if osp.isdir(directory) else [] + num_deleted = 0 for name in names: - os.remove(osp.join(directory, name)) + try: + os.remove(osp.join(directory, name)) + num_deleted += 1 + except FileNotFoundError: + pass # deleted by another process - if osp.isdir(directory) and not os.listdir(directory): + try: os.rmdir(directory) + except OSError: + pass # not empty, or removed by another process - return len(names) + return num_deleted def find_pack(filepath: PathLike) -> str | None: diff --git a/tests/common/test_t4pack.py b/tests/common/test_t4pack.py index e9311cf4..81a11717 100644 --- a/tests/common/test_t4pack.py +++ b/tests/common/test_t4pack.py @@ -1,5 +1,7 @@ from __future__ import annotations +import multiprocessing +import os from pathlib import Path import numpy as np @@ -260,6 +262,60 @@ def test_delete_loose_frames_requires_pack(tmp_path: Path) -> None: assert (directory / "00000.pcd.bin").exists() +def _pack_concurrently(directory: str) -> None: + pack_directory(directory, level=1, overwrite=True) + + +def _delete_concurrently(directory: str) -> int: + return delete_loose_frames(directory) + + +def test_pack_directory_concurrently(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + names = [f"{i:05d}.pcd.bin" for i in range(20)] + _write_frames(directory, [_frame(5000, seed=i) for i in range(len(names))], names) + + with multiprocessing.get_context("spawn").Pool(4) as pool: + pool.map(_pack_concurrently, [str(directory)] * 4) + + assert verify_pack(directory) == [] + assert sorted(p.name for p in tmp_path.iterdir()) == ["LIDAR_CONCAT", "LIDAR_CONCAT.pack"] + + with multiprocessing.get_context("spawn").Pool(4) as pool: + num_deleted = pool.map(_delete_concurrently, [str(directory)] * 4) + + assert sum(num_deleted) == len(names) + assert not directory.exists() + assert verify_pack(directory) == [] + + +def test_pack_directory_keeps_default_permissions(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + _write_frames(directory, [_frame(10)], ["00000.pcd.bin"]) + + pack_directory(directory, level=1) + + umask = os.umask(0) + os.umask(umask) + assert (tmp_path / "LIDAR_CONCAT.pack").stat().st_mode & 0o777 == 0o666 & ~umask + + +def test_pack_directory_refuses_to_drop_deleted_frames(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + names = ["00000.pcd.bin", "00001.pcd.bin"] + frames = [_frame(100, seed=i) for i in range(len(names))] + _write_frames(directory, frames, names) + pack_directory(directory, level=1) + + # partially deleted, e.g. interrupted deletion + (directory / names[0]).unlink() + + with pytest.raises(ValueError): + pack_directory(directory, level=1, overwrite=True) + with PackReader(tmp_path / "LIDAR_CONCAT.pack") as reader: + assert np.array_equal(reader.read_frame_by_name(names[0]), frames[0]) + + def test_find_pack(tmp_path: Path) -> None: filepath = tmp_path / "LIDAR_CONCAT" / "00000.pcd.bin" From 9d5b3e354f5a4994ba0618971fb4065e8406d1e8 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Fri, 23 Oct 2026 10:31:05 +0900 Subject: [PATCH 09/12] fix(common): compare frames bit by bit and report failed channels in t4pack pack Co-Authored-By: Claude Opus 5.5 --- docs/cli/t4pack.md | 1 + t4_devkit/cli/pack.py | 37 ++++++++++++++++++------------ t4_devkit/common/t4pack.py | 23 +++++++++++++++---- t4_devkit/dataclass/pointcloud.py | 5 +++- tests/cli/test_pack.py | 17 +++++++++++++- tests/common/test_t4pack.py | 28 ++++++++++++++++++++++ tests/dataclass/test_pointcloud.py | 10 ++++++++ 7 files changed, 100 insertions(+), 21 deletions(-) diff --git a/docs/cli/t4pack.md b/docs/cli/t4pack.md index a3b9de31..4ae3b7cc 100644 --- a/docs/cli/t4pack.md +++ b/docs/cli/t4pack.md @@ -58,6 +58,7 @@ t4pack pack Every frame is verified bit-exact against its source before the pack is put in place, so an existing pack is always complete. If the source values do not fit the compact `uint8`/`int8` columns, e.g. non-integer intensity, every column is stored as `float32` instead. A channel that is already packed is skipped unless `--overwrite` is specified. +If a channel fails, it is reported as `FAILED` and the other channels are still processed, then the command exits with `1`. The source `.pcd.bin` files are kept by default. With `--delete-loose`, they are deleted after every frame is verified against the pack again, which also applies to a channel that is already packed: diff --git a/t4_devkit/cli/pack.py b/t4_devkit/cli/pack.py index 5f3e5073..cd35144c 100644 --- a/t4_devkit/cli/pack.py +++ b/t4_devkit/cli/pack.py @@ -71,25 +71,32 @@ def pack( ) -> None: channel_dirs = _find_channel_dirs(data_root, revision) + is_passed = True for channel_dir in channel_dirs: num_pts_feats = _num_pts_feats(channel_dir) - stats = pack_directory( - channel_dir, - num_pts_feats=num_pts_feats, - level=level, - overwrite=overwrite, - ) - if stats is None: - console.print(f"[yellow]SKIP[/yellow] {channel_dir}: already packed") - else: - console.print( - f"[green]PACKED[/green] {channel_dir}: {stats.num_frames} frames, " - f"{stats.raw_bytes} -> {stats.packed_bytes} bytes ({stats.ratio:.2f}x)" + try: + stats = pack_directory( + channel_dir, + num_pts_feats=num_pts_feats, + level=level, + overwrite=overwrite, ) + if stats is None: + console.print(f"[yellow]SKIP[/yellow] {channel_dir}: already packed") + else: + console.print( + f"[green]PACKED[/green] {channel_dir}: {stats.num_frames} frames, " + f"{stats.raw_bytes} -> {stats.packed_bytes} bytes ({stats.ratio:.2f}x)" + ) + + if delete_loose: + num_deleted = delete_loose_frames(channel_dir, num_pts_feats=num_pts_feats) + console.print(f"[green]DELETED[/green] {channel_dir}: {num_deleted} frames") + except (OSError, ValueError) as e: + is_passed = False + console.print(f"[red]FAILED[/red] {channel_dir}: {e}") - if delete_loose: - num_deleted = delete_loose_frames(channel_dir, num_pts_feats=num_pts_feats) - console.print(f"[green]DELETED[/green] {channel_dir}: {num_deleted} frames") + sys.exit(0 if is_passed else 1) @cli.command( diff --git a/t4_devkit/common/t4pack.py b/t4_devkit/common/t4pack.py index 194669bb..016706e6 100644 --- a/t4_devkit/common/t4pack.py +++ b/t4_devkit/common/t4pack.py @@ -324,6 +324,9 @@ def __init__(self, filepath: PathLike) -> None: def __len__(self) -> int: return len(self.frames) + def __contains__(self, name: str) -> bool: + return name in self._name_to_index + def __enter__(self) -> Self: return self @@ -376,7 +379,7 @@ def read_frame_by_name(self, name: str) -> NDArrayF32: Returns: NDArrayF32: Points in the shape of (N, num_columns). """ - if name not in self._name_to_index: + if name not in self: raise KeyError(f"Frame '{name}' is not found in {self.filepath}") return self.read_frame(self._name_to_index[name]) @@ -456,11 +459,14 @@ def pack_directory( filepaths = [osp.join(directory, name) for name in names] layout = PackLayout.from_num_pts_feats(num_pts_feats) + lossless_layout = layout.to_lossless() + # try the compact layout first, and fall back to the lossless one if it does not round-trip + candidates = [layout] if layout == lossless_layout else [layout, lossless_layout] # write to a temporary file, verify it, and then rename it to the final path tmp_filepath = _make_tmp_file(pack_filepath) try: - for candidate in dict.fromkeys((layout, layout.to_lossless())): + for candidate in candidates: num_frames = write_pack( tmp_filepath, (_load_frame(filepath, num_pts_feats) for filepath in filepaths), @@ -471,7 +477,8 @@ def pack_directory( mismatches = _compare_frames(tmp_filepath, directory, names, num_pts_feats) if not mismatches: break - else: + + if mismatches: raise ValueError( f"{len(mismatches)} frames differ after packing {directory}, e.g. {mismatches[:3]}" ) @@ -534,11 +541,19 @@ def _compare_frames( source = _load_frame(osp.join(directory, name), num_pts_feats) except FileNotFoundError: continue # deleted by another process after listing, the frame is decodable - if not np.array_equal(packed, source): + if not _is_bit_exact(packed, source): mismatches.append(name) return mismatches +def _is_bit_exact(lhs: NDArrayF32, rhs: NDArrayF32) -> bool: + """Return whether two arrays are identical bit by bit. + + Unlike `np.array_equal`, NaN equals NaN and -0.0 differs from 0.0. + """ + return lhs.shape == rhs.shape and lhs.tobytes() == rhs.tobytes() + + def delete_loose_frames(directory: PathLike, num_pts_feats: int = 5) -> int: """Delete the `.pcd.bin` frames in a directory after verifying `.pack`. diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index a6464978..8880df61 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -43,9 +43,12 @@ def _load_pcd_bin(filepath: str, num_pts_feats: int) -> NDArrayFloat: NDArrayFloat: Points in the shape of (N, num_pts_feats). """ if not osp.exists(filepath) and (pack_filepath := find_pack(filepath)) is not None: + name = osp.basename(filepath) with PackReader(pack_filepath) as reader: - return reader.read_frame_by_name(osp.basename(filepath)) + if name in reader: + return reader.read_frame_by_name(name) + # raise FileNotFoundError if the frame is neither a file nor in the pack scan = np.fromfile(filepath, dtype=np.float32) return scan.reshape((-1, num_pts_feats)) diff --git a/tests/cli/test_pack.py b/tests/cli/test_pack.py index fb106611..9c25bd59 100644 --- a/tests/cli/test_pack.py +++ b/tests/cli/test_pack.py @@ -76,10 +76,25 @@ def test_delete_loose_refuses_changed_frame(tmp_path: Path) -> None: np.ones((10, 5), dtype=np.float32).tofile(lidar_dir / "00001.pcd.bin") result = runner.invoke(cli, ["pack", str(data_root), "--delete-loose"]) - assert result.exit_code != 0 + assert result.exit_code == 1 + assert "FAILED" in result.output assert len(list(lidar_dir.glob("*.pcd.bin"))) == 3 +def test_pack_continues_after_failed_channel(tmp_path: Path) -> None: + data_root = _create_dataset(tmp_path) + data_dir = data_root / "0" / "data" + # a broken channel sorted before LIDAR_CONCAT + (data_dir / "LIDAR_BROKEN").mkdir() + np.zeros(7, dtype=np.float32).tofile(data_dir / "LIDAR_BROKEN" / "00000.pcd.bin") + + result = CliRunner().invoke(cli, ["pack", str(data_root), "--level", "1"]) + + assert result.exit_code == 1 + assert "FAILED" in result.output + assert (data_dir / "LIDAR_CONCAT.pack").exists() + + def test_verify_fails_on_changed_frame(tmp_path: Path) -> None: data_root = _create_dataset(tmp_path) runner = CliRunner() diff --git a/tests/common/test_t4pack.py b/tests/common/test_t4pack.py index 81a11717..24bbc47d 100644 --- a/tests/common/test_t4pack.py +++ b/tests/common/test_t4pack.py @@ -91,6 +91,8 @@ def test_write_and_read_pack(tmp_path: Path) -> None: reader.read_frame(3) with pytest.raises(KeyError): reader.read_frame_by_name("2.pcd.bin") + assert "10.pcd.bin" in reader + assert "2.pcd.bin" not in reader def test_write_pack_names_frames_by_index_by_default(tmp_path: Path) -> None: @@ -176,6 +178,32 @@ def test_pack_directory_falls_back_to_lossless_layout(tmp_path: Path) -> None: assert verify_pack(directory) == [] +def test_pack_directory_keeps_nan_points(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + frame = _frame(100) + frame[0, :3] = np.nan + _write_frames(directory, [frame], ["00000.pcd.bin"]) + + pack_directory(directory, level=1) + + assert verify_pack(directory) == [] + with PackReader(tmp_path / "LIDAR_CONCAT.pack") as reader: + assert reader.read_frame(0).tobytes() == frame.tobytes() + + +def test_pack_directory_is_bit_exact_for_negative_zero(tmp_path: Path) -> None: + directory = tmp_path / "LIDAR_CONCAT" + frame = _frame(100) + frame[0, 3] = -0.0 # equals 0.0, but is lost by uint8 + _write_frames(directory, [frame], ["00000.pcd.bin"]) + + pack_directory(directory, level=1) + + with PackReader(tmp_path / "LIDAR_CONCAT.pack") as reader: + assert reader.frames[0]["dtypes"] == ["f4s"] * 5 + assert reader.read_frame(0).tobytes() == frame.tobytes() + + def test_pack_layout_to_lossless() -> None: layout = PackLayout.from_num_pts_feats(5).to_lossless() diff --git a/tests/dataclass/test_pointcloud.py b/tests/dataclass/test_pointcloud.py index 2af11025..dd9c5740 100644 --- a/tests/dataclass/test_pointcloud.py +++ b/tests/dataclass/test_pointcloud.py @@ -443,6 +443,16 @@ def test_lidar_pointcloud_from_file_raises_without_file_and_pack(tmp_path: Path) LidarPointCloud.from_file(str(tmp_path / "LIDAR_CONCAT" / "00000.pcd.bin")) +def test_lidar_pointcloud_from_file_raises_if_frame_is_not_in_pack(tmp_path: Path) -> None: + lidar_dir = tmp_path / "data" / "LIDAR_CONCAT" + lidar_dir.mkdir(parents=True) + np.zeros((3, 5), dtype=np.float32).tofile(lidar_dir / "00000.pcd.bin") + pack_directory(lidar_dir, level=1) + + with pytest.raises(FileNotFoundError): + LidarPointCloud.from_file(str(lidar_dir / "00001.pcd.bin")) + + def test_segmentation_pointcloud_from_file_reads_points_from_pack(tmp_path: Path) -> None: lidar_dir = tmp_path / "data" / "LIDAR_CONCAT" lidar_dir.mkdir(parents=True) From 6746d9561d7276fafdb6e554768922f61e5b11ef Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Fri, 23 Oct 2026 10:33:57 +0900 Subject: [PATCH 10/12] docs(common): remove reference to internal reader from t4pack docstring Co-Authored-By: Claude Opus 5.5 --- t4_devkit/common/t4pack.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/t4_devkit/common/t4pack.py b/t4_devkit/common/t4pack.py index 016706e6..efee9da1 100644 --- a/t4_devkit/common/t4pack.py +++ b/t4_devkit/common/t4pack.py @@ -14,8 +14,6 @@ byte-shuffled (dtype code `f4s`): the four bytes of every float are split into four planes, so the near-constant sign/exponent bytes are grouped together and zstd can match them. Each frame is compressed independently, so reading one frame decodes only that frame. - -The format is compatible with the e2e-devkit `lidar_pack.py` reader. """ from __future__ import annotations From 87801ffef671de286e4faa26d34a09585aa4014c Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Fri, 23 Oct 2026 10:37:34 +0900 Subject: [PATCH 11/12] refactor: replace default num_pts_feats literal with DEFAULT_NUM_PTS_FEATS Co-Authored-By: Claude Opus 5.5 --- t4_devkit/cli/pack.py | 5 +++-- t4_devkit/common/t4pack.py | 10 +++++++--- t4_devkit/dataclass/pointcloud.py | 14 +++++++++----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/t4_devkit/cli/pack.py b/t4_devkit/cli/pack.py index cd35144c..647ce2e0 100644 --- a/t4_devkit/cli/pack.py +++ b/t4_devkit/cli/pack.py @@ -10,6 +10,7 @@ from rich.console import Console from t4_devkit.common.t4pack import ( + DEFAULT_NUM_PTS_FEATS, FRAME_SUFFIX, PACK_SUFFIX, delete_loose_frames, @@ -161,8 +162,8 @@ def _find_packed_channel_dirs(data_root: str, revision: str | None) -> list[str] def _num_pts_feats(channel_dir: str) -> int: - """Return `num_pts_feats` from the metainfo in `_INFO/`, or 5 if there is none.""" + """Return `num_pts_feats` from the metainfo in `_INFO/`, or the default if none.""" info_filepaths = sorted(glob(osp.join(f"{channel_dir}_INFO", "*.json"))) if not info_filepaths: - return 5 + return DEFAULT_NUM_PTS_FEATS return PointCloudMetainfo.from_file(info_filepaths[0]).num_pts_feats diff --git a/t4_devkit/common/t4pack.py b/t4_devkit/common/t4pack.py index efee9da1..36834f56 100644 --- a/t4_devkit/common/t4pack.py +++ b/t4_devkit/common/t4pack.py @@ -36,6 +36,8 @@ __all__ = ( "PACK_SUFFIX", + "FRAME_SUFFIX", + "DEFAULT_NUM_PTS_FEATS", "PackLayout", "PackReader", "PackStats", @@ -55,6 +57,8 @@ PACK_SUFFIX = ".pack" #: Suffix of a loose pointcloud frame. FRAME_SUFFIX = ".pcd.bin" +#: Default number of float32 fields in each point of a frame: (x, y, z, intensity, ring). +DEFAULT_NUM_PTS_FEATS = 5 @define(frozen=True) @@ -407,7 +411,7 @@ def _list_frames(directory: str) -> list[str]: def pack_directory( directory: PathLike, - num_pts_feats: int = 5, + num_pts_feats: int = DEFAULT_NUM_PTS_FEATS, level: int = 19, overwrite: bool = False, ) -> PackStats | None: @@ -492,7 +496,7 @@ def pack_directory( ) -def verify_pack(directory: PathLike, num_pts_feats: int = 5) -> list[str]: +def verify_pack(directory: PathLike, num_pts_feats: int = DEFAULT_NUM_PTS_FEATS) -> list[str]: """Verify `.pack` against the `.pcd.bin` frames in the directory. A frame whose source exists must equal it bit-exact, a source frame must be in the pack, @@ -552,7 +556,7 @@ def _is_bit_exact(lhs: NDArrayF32, rhs: NDArrayF32) -> bool: return lhs.shape == rhs.shape and lhs.tobytes() == rhs.tobytes() -def delete_loose_frames(directory: PathLike, num_pts_feats: int = 5) -> int: +def delete_loose_frames(directory: PathLike, num_pts_feats: int = DEFAULT_NUM_PTS_FEATS) -> int: """Delete the `.pcd.bin` frames in a directory after verifying `.pack`. Nothing is deleted unless every frame in the directory equals its frame in the pack, diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index 8880df61..1f03e695 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -10,7 +10,7 @@ from attrs import define, field from t4_devkit.common.io import load_json -from t4_devkit.common.t4pack import PackReader, find_pack +from t4_devkit.common.t4pack import DEFAULT_NUM_PTS_FEATS, PackReader, find_pack if TYPE_CHECKING: from typing_extensions import Self @@ -104,7 +104,7 @@ class PointCloudMetainfo: stamp: Stamp = field(converter=lambda x: Stamp(**x) if isinstance(x, dict) else x) sources: list[PointCloudSourceInfo] = field(factory=list) - num_pts_feats: int = 5 + num_pts_feats: int = DEFAULT_NUM_PTS_FEATS @classmethod def from_file(cls, filepath: str) -> Self: @@ -121,7 +121,11 @@ def from_file(cls, filepath: str) -> Self: sources = [] for source_data in data.get("sources", []): sources.append(PointCloudSourceInfo(**source_data)) - return cls(stamp=stamp, sources=sources, num_pts_feats=data.get("num_pts_feats", 5)) + return cls( + stamp=stamp, + sources=sources, + num_pts_feats=data.get("num_pts_feats", DEFAULT_NUM_PTS_FEATS), + ) @property def source_tokens(self) -> list[str]: @@ -296,7 +300,7 @@ def from_file(cls, filepath: str, metainfo_filepath: str | None = None) -> Self: ) if filepath.endswith(".bin"): - num_pts_feats = getattr(metainfo, "num_pts_feats", 5) + num_pts_feats = getattr(metainfo, "num_pts_feats", DEFAULT_NUM_PTS_FEATS) points = _load_pcd_bin(filepath, num_pts_feats)[:, : cls.num_dims()] else: pcd = pypcd4.PointCloud.from_path(filepath) @@ -447,7 +451,7 @@ def from_file( ) if point_filepath.endswith(".bin"): - num_pts_feats = getattr(metainfo, "num_pts_feats", 5) + num_pts_feats = getattr(metainfo, "num_pts_feats", DEFAULT_NUM_PTS_FEATS) points = _load_pcd_bin(point_filepath, num_pts_feats)[:, : cls.num_dims()] else: pcd = pypcd4.PointCloud.from_path(point_filepath) From beec954a828fcd67cc1fe24e7e008024f95673e0 Mon Sep 17 00:00:00 2001 From: Jian Kang Date: Fri, 23 Oct 2026 15:05:21 +0900 Subject: [PATCH 12/12] style: remove extra blank line in REF201 tests Co-Authored-By: Claude Opus 5.5 --- tests/sanity/test_reference_checkers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/sanity/test_reference_checkers.py b/tests/sanity/test_reference_checkers.py index c32081ed..2777da4f 100644 --- a/tests/sanity/test_reference_checkers.py +++ b/tests/sanity/test_reference_checkers.py @@ -320,7 +320,6 @@ def test_ref201_fail_missing_filename(tmp_path: Path) -> None: assert any("does_not_exist.jpg" in r for r in report.reasons) - def test_ref201_pass_packed_frames(tmp_path: Path) -> None: """REF201 passes if loose .pcd.bin frames are deleted but stored in their pack.""" root = _copy_dataset(tmp_path / "dataset_ref201_packed")