From 940c5504d70ae238debe9de20e0724209e3f6eda Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sat, 15 Aug 2026 19:58:29 +0100 Subject: [PATCH 1/5] Preserve the device module type in DatasetReader DatasetReader is now generic over its device module, so device_module returns the module passed to the constructor rather than the DeviceModuleLike contract, and create_dataset_reader declares DatasetReader[DeviceModule]. Reaching a register through the device module in the reader now resolves to the register class itself for a pre-generated package, and collectively for a module built by create_device_module. --- .../harp-data/src/harp/data/_dataset.py | 24 ++++++++++++----- tests/conformance.py | 27 ++++++++++++++++++- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index bb8a49b..f7ac8a3 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -3,15 +3,17 @@ from datetime import datetime from os import PathLike from pathlib import Path -from typing import Any +from typing import Any, Generic, TypeVar import pandas as pd -from harp.device.schema import DeviceModuleLike, create_device_module +from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module from harp.protocol import RegisterBase from harp.protocol._constants import _TIMESTAMP_FLAG from ._reader import parse_to_dataframe +M = TypeVar("M", bound=DeviceModuleLike) + RegisterKey = type[RegisterBase[Any]] | int FileNameResolver = Callable[[Path, str], Mapping[int, list[Path]]] @@ -31,7 +33,7 @@ def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: return files -class DatasetReader: +class DatasetReader(Generic[M]): """Reader over a de-multiplexed Harp dataset folder. Construct from a device module and a dataset folder, then read the @@ -47,6 +49,9 @@ class DatasetReader: ``__name__`` are read on demand. ``name`` overrides the ```` file prefix, which defaults to the module name. + The reader is typed on the module it was given, so registers stay reachable through + :attr:`device_module` at whatever precision that module offers. + File resolution defaults to the Harp file format: ``_
.bin`` and, when a register was logged as several ``_
_.bin`` chunks, they are concatenated in filename order. Pass ``resolver`` (a :data:`FileResolver`) @@ -55,7 +60,7 @@ class DatasetReader: def __init__( self, - device_module: DeviceModuleLike, + device_module: M, root: str | PathLike[str], *, name: str | None = None, @@ -73,8 +78,13 @@ def root(self) -> Path: return self._root @property - def device_module(self) -> DeviceModuleLike: - """The device module this reader parses against.""" + def device_module(self) -> M: + """The device module this reader parses against, as the type it was given. + + A generated package resolves each register to its own class; one built by + :func:`~harp.device.schema.create_device_module` resolves them collectively, + the same ceiling as reaching it directly. + """ return self._device_module @property @@ -193,7 +203,7 @@ def create_dataset_reader( resolver: FileNameResolver = default_file_resolver, converters: Mapping[str, Any] | None = None, strict: bool = True, -) -> DatasetReader: +) -> DatasetReader[DeviceModule]: """Build a :class:`DatasetReader` for a dataset folder, device and all. Convenience wrapper that finds the device schema inside ``root`` (``device.yml`` diff --git a/tests/conformance.py b/tests/conformance.py index bea71e7..d0cc2f8 100644 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -8,7 +8,7 @@ from typing import Any, ClassVar, assert_type import numpy as np -from harp.data import DatasetReader +from harp.data import DatasetReader, create_dataset_reader from harp.device.client import Device, ITransport from harp.device.core import OperationControl, OperationControlPayload, WhoAmI from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module @@ -85,6 +85,31 @@ def dataset_reader_accepts_either_module( reader.read(44) +def dataset_reader_keeps_module_type( + schema_built: DeviceModule, generated: DeviceModuleLike +) -> None: + """The reader is typed on the module it was given, not on the contract. + + Reading it back as ``DeviceModuleLike`` would leave only the three declarations of + the contract, so every register reached through the reader would fail to resolve. + """ + assert_type(DatasetReader(schema_built, "session.harp").device_module, DeviceModule) + assert_type(DatasetReader(generated, "session.harp").device_module, DeviceModuleLike) + + +def dataset_reader_registers_resolve_through_the_module() -> None: + """A register stays reachable through the reader, at the precision of its module. + + A schema-built module resolves collectively, as it does when reached directly, so + the ceiling here is the one :func:`create_device_module` documents. A generated + package carries its own declarations and resolves each to its own class. + """ + reader = create_dataset_reader("session.harp") + assert_type(reader, DatasetReader[DeviceModule]) + assert_type(reader.device_module.AnalogData, Any) + reader.read(reader.device_module.AnalogData) + + def open_serial_device_prefers_the_subclass_overload() -> None: """A Device subclass is matched as a subclass even when it looks like a module. From 6ee84933972baa2a25f683122b0cffc373b51dc0 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 16 Aug 2026 00:05:32 +0100 Subject: [PATCH 2/5] Match dataset files by the declared device name A device module now declares DEVICE_NAME, the device name from the schema, matching what the generator emits for a static package, and DeviceModuleLike requires it in place of __name__. DatasetReader takes the file prefix from name when given, then from DEVICE_NAME, then from the folder when it holds a single device. A pre-generated package therefore reads correctly, where the module name used before is its dotted import path and matches no file. --- src/packages/harp-data/README.md | 17 +++--- .../harp-data/src/harp/data/_dataset.py | 38 ++++++++++-- .../src/harp/device/client/_device.py | 2 +- .../src/harp/device/schema/_module.py | 33 ++++++---- tests/conformance.py | 8 ++- tests/data/test_dataset.py | 60 +++++++++++++++++++ tests/device/expected_device.py | 2 + tests/device/test_create_device_module.py | 16 +++-- tests/device/test_device.py | 1 + 9 files changed, 146 insertions(+), 31 deletions(-) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index accb8b4..e1460a3 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -25,24 +25,25 @@ Reading is based on a [device module](../harp-device) that describes how to deco from harp import data reader = data.create_dataset_reader("session.harp") -df = reader.read(AnalogData) # by register class -df = reader.read(44) # by address -everything = reader.read_all() # {register_name: DataFrame} +behavior = reader.device_module +df = reader.read(behavior.AnalogData) # by register class +df = reader.read(44) # by address +everything = reader.read_all() # {register_name: DataFrame} ``` Given a device module already in hand, either a pre-generated package or one built with `create_device_module`, pass it to `DatasetReader` directly: ```python -from pathlib import Path - from harp import data -from harp.device import schema +from harp.device import behavior -behavior = schema.create_device_module((Path("session.harp") / "device.yml").read_bytes()) reader = data.DatasetReader(behavior, "session.harp") +df = reader.read(behavior.AnalogData) ``` -Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout, or `name=` to override the file prefix. +Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. + +The `` prefix comes from the `DEVICE_NAME` the device module declares. When there is no declared name, the prefix is read off the folder instead. Pass `name=` to override it, either deliberately or to fix a folder the rule cannot resolve. ## Read a single register file diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index f7ac8a3..f69db13 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -22,6 +22,14 @@ """Default filename of the device schema looked up inside a dataset folder.""" +_FILE_STEM = re.compile(r"^(?P.+?)_(?P
\d+)(?:_.*)?$") + + +def _discover_device_names(root: Path) -> set[str]: + stems = (_FILE_STEM.match(path.stem) for path in root.glob("*.bin")) + return {match.group("name") for match in stems if match is not None} + + def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: """Harp file format resolver: map address -> sorted ``_
...`` files.""" pattern = re.compile(rf"^{re.escape(name)}_(\d+)(?:_.*)?$") @@ -45,9 +53,12 @@ class DatasetReader(Generic[M]): everything = reader.read_all() # {register_name: DataFrame} ``device_module`` is a device module -- a generated device package, or one built from a - schema with :func:`~harp.device.schema.create_device_module`. Its ``REGISTER_MAP`` and - ``__name__`` are read on demand. ``name`` overrides the ```` file - prefix, which defaults to the module name. + schema with :func:`~harp.device.schema.create_device_module`. Its ``REGISTER_MAP`` is + read on demand. + + The files are matched by a ```` prefix, taken from the ``DEVICE_NAME`` + the module declares, or read off the folder when it declares none. Pass ``name`` to + override that, either deliberately or to read a folder the rule cannot resolve. The reader is typed on the module it was given, so registers stay reachable through :attr:`device_module` at whatever precision that module offers. @@ -70,7 +81,8 @@ def __init__( self._root = Path(root) self._name_override = name self._resolver = resolver - self._files = dict(self._resolver(self._root, self.name)) + self._name = self._resolve_name() + self._files = dict(self._resolver(self._root, self._name)) @property def root(self) -> Path: @@ -90,7 +102,23 @@ def device_module(self) -> M: @property def name(self) -> str: """The ```` prefix used to match binary files.""" - return self._name_override or self._device_module.__name__ + return self._name + + def _resolve_name(self) -> str: + if self._name_override is not None: + return self._name_override + declared = getattr(self._device_module, "DEVICE_NAME", "") + if declared: + return declared + found = _discover_device_names(self._root) + if len(found) == 1: + return found.pop() + raise ValueError( + f"The device module declares no DEVICE_NAME, so the file prefix has to come " + f"from {self._root}, which holds " + + (f"data for {sorted(found)}" if found else "no Harp data files") + + ". Pass name= with the prefix to read." + ) @property def registers(self) -> Mapping[int, type[RegisterBase[Any]]]: diff --git a/src/packages/harp-device/src/harp/device/client/_device.py b/src/packages/harp-device/src/harp/device/client/_device.py index 6734ddc..0bcbb9c 100644 --- a/src/packages/harp-device/src/harp/device/client/_device.py +++ b/src/packages/harp-device/src/harp/device/client/_device.py @@ -167,7 +167,7 @@ def _validate_whoami(self) -> None: actual = int(self.read(WhoAmI).parsed) if actual != expected: raise RuntimeError( - f"WhoAmI mismatch: {module.__name__} expects 0x{expected:04x} " + f"WhoAmI mismatch: {module.DEVICE_NAME} expects 0x{expected:04x} " f"but the device reported 0x{actual:04x}." ) diff --git a/src/packages/harp-device/src/harp/device/schema/_module.py b/src/packages/harp-device/src/harp/device/schema/_module.py index 4c1f422..aa7f989 100644 --- a/src/packages/harp-device/src/harp/device/schema/_module.py +++ b/src/packages/harp-device/src/harp/device/schema/_module.py @@ -27,27 +27,34 @@ class DeviceModuleLike(Protocol): what identifies it is describing a device. Matching structurally accepts both it and :class:`DeviceModule`, and rejects the common register set, which carries registers but is not a device. + + ``DEVICE_NAME`` is required rather than optional, so a generated package always + states the name used for its recordings. A schema declaring none still + builds, since :class:`DeviceModule` declares the member and leaves it empty. """ - __name__: str - REGISTER_MAP: dict[int, type[RegisterBase[Any]]] + DEVICE_NAME: str WHO_AM_I: int + REGISTER_MAP: dict[int, type[RegisterBase[Any]]] class DeviceModule(types.ModuleType): """The type of the module returned by :func:`create_device_module`. The declarations of the schema are reached by name and typed ``Any``, since they - exist only at runtime. ``REGISTER_MAP``, ``WHO_AM_I`` and ``__all__`` are declared - here and carry their own types. + exist only at runtime. ``DEVICE_NAME``, ``REGISTER_MAP``, ``WHO_AM_I`` and + ``__all__`` are declared here and carry their own types. """ - REGISTER_MAP: dict[int, type[RegisterBase[Any]]] - """Address -> register class, the common Harp registers merged with those of the schema.""" + DEVICE_NAME: str + """The device name declared by the schema. Empty when absent.""" WHO_AM_I: int """The device identity declared by the schema. ``0`` when absent.""" + REGISTER_MAP: dict[int, type[RegisterBase[Any]]] + """Address -> register class, the common Harp registers merged with those of the schema.""" + __all__: list[str] """The declarations of the schema, beside ``REGISTER_MAP`` and ``WHO_AM_I``.""" @@ -72,8 +79,12 @@ def create_device_module( * ``REGISTER_MAP``, the device address space, so the common registers are present here even though the module does not name them; * ``WHO_AM_I``, the identity declared by the schema (``0`` for an unregistered device); - * ``__name__``, the ``device`` name of the schema, or ``name`` when given - (``"Device"`` for a header-less register fragment). + * ``DEVICE_NAME``, the ``device`` name of the schema, or ``name`` when given, and + empty for a header-less register fragment. Recordings are written under this + name, so :class:`~harp.data.DatasetReader` matches files by it; + * ``__name__``, the same name, falling back to ``"Device"`` so the module is never + anonymous. This names the module rather than the device, and is not part of what + a device module promises. Because the names come from the schema at runtime they don't autocomplete, and each resolves as ``Any`` rather than its own type. A generated device package is @@ -91,7 +102,8 @@ def create_device_module( device = parse_device_schema(text) emitter = _Emitter(device, converters, strict) registers = emitter.emit() - module_name = name or device.device or _DEFAULT_NAME + device_name = name or device.device or "" + module_name = device_name or _DEFAULT_NAME contents: dict[str, Any] = {**emitter.enums, **emitter.payloads, **registers} register_map = {cls.address: cls for cls in CORE_REGISTER_MAP.values()} @@ -103,8 +115,9 @@ def create_device_module( module = DeviceModule(module_name, f"Harp registers for {module_name}, from a schema.") vars(module).update( contents, + DEVICE_NAME=device_name, REGISTER_MAP=register_map, WHO_AM_I=int(device.whoAmI or 0), - __all__=[*sorted(contents), "REGISTER_MAP", "WHO_AM_I"], + __all__=[*sorted(contents), "DEVICE_NAME", "REGISTER_MAP", "WHO_AM_I"], ) return module diff --git a/tests/conformance.py b/tests/conformance.py index d0cc2f8..4fe587c 100644 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -114,13 +114,15 @@ def open_serial_device_prefers_the_subclass_overload() -> None: """A Device subclass is matched as a subclass even when it looks like a module. type[D] is narrower than the structural module overload, so it has to come first: - a class carrying REGISTER_MAP and WHO_AM_I satisfies DeviceModuleLike too, and the - module overload would otherwise win and return Device[type[Hybrid]]. + a class carrying the members of DeviceModuleLike satisfies it too, and the module + overload would otherwise win and return Device[type[Hybrid]]. The class has to + carry every member for this to test the ordering rather than the match. """ class Hybrid(Device[None]): - REGISTER_MAP: ClassVar[dict[int, type[RegisterBase[Any]]]] = {} + DEVICE_NAME: ClassVar[str] = "Hybrid" WHO_AM_I: ClassVar[int] = 1216 + REGISTER_MAP: ClassVar[dict[int, type[RegisterBase[Any]]]] = {} device = open_serial_device(Hybrid, port="COM3") assert_type(device, Hybrid) diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index fe1d733..0eb58a5 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -152,6 +152,66 @@ def test_explicit_name_overrides(dataset): assert reader.name == name +def test_name_from_device_name_not_module_name(dataset): + # The prefix follows DEVICE_NAME rather than __name__, so rebinding the module + # does not change which files are read. + mod, name, root, _specs = dataset + mod.__name__ = "not_the_device_name" + assert DatasetReader(mod, root).name == name + + +def test_name_discovered_without_device_name(dataset): + # A device package published before DEVICE_NAME existed declares no name, so the + # folder is the only remaining source. + mod, name, root, _specs = dataset + del mod.DEVICE_NAME + assert DatasetReader(mod, root).name == name + + +def test_ambiguous_folder_raises(dataset): + mod, name, root, specs = dataset + del mod.DEVICE_NAME + address = next(iter(specs)) + (root / f"Other_{address}.bin").write_bytes(b"") + with pytest.raises(ValueError, match="Pass name=") as excinfo: + DatasetReader(mod, root) + # The message names what it found, so the caller knows what to choose between. + assert name in str(excinfo.value) + assert "Other" in str(excinfo.value) + assert DatasetReader(mod, root, name=name).name == name + + +def test_empty_dataset_reads_empty(emitted_module, tmp_path): + # A session that logged nothing is a dataset with no data, not a failure. + reader = DatasetReader(emitted_module, tmp_path) + assert reader.name == emitted_module.DEVICE_NAME + assert reader.files == {} + assert reader.read_all() == {} + + +def test_unresolvable_prefix_raises(emitted_module, tmp_path): + # Nothing declares a name and nothing on disk suggests one, so no prefix can be + # determined. This is the reader failing to be set up, not an empty dataset. + del emitted_module.DEVICE_NAME + with pytest.raises(ValueError, match="Pass name="): + DatasetReader(emitted_module, tmp_path) + + +def test_chunked_files_discover_single_name(emitted_module, tmp_path): + # A stem splits at its first address, so the chunk suffix of a multi-chunk register + # stays out of the device name and one device is not discovered as several. + mod = emitted_module + name = mod.DEVICE_NAME + address = next(a for a in sorted(mod.REGISTER_MAP) if a >= 32) + cls = mod.REGISTER_MAP[address] + for chunk in range(2): + buf = bytes(cls.format_bulk(_records(cls, 2, seed=chunk))) + (tmp_path / f"{name}_{address}_{chunk}.bin").write_bytes(buf) + + del mod.DEVICE_NAME + assert DatasetReader(mod, tmp_path).name == name + + def test_missing_register_file_raises(dataset): mod, _name, root, _specs = dataset reader = DatasetReader(mod, root) diff --git a/tests/device/expected_device.py b/tests/device/expected_device.py index db3371c..0f027f3 100644 --- a/tests/device/expected_device.py +++ b/tests/device/expected_device.py @@ -31,6 +31,7 @@ __all__ = [ + "DEVICE_NAME", "WHO_AM_I", "PortDigitalIOS", "PwmPort", @@ -64,6 +65,7 @@ "REGISTER_MAP", ] +DEVICE_NAME: str = "Tests" WHO_AM_I: int = 0 diff --git a/tests/device/test_create_device_module.py b/tests/device/test_create_device_module.py index 5fa8653..ae8c39f 100644 --- a/tests/device/test_create_device_module.py +++ b/tests/device/test_create_device_module.py @@ -12,7 +12,7 @@ from .converters import DataConverter CONVERTERS = {"DataConverter": DataConverter()} -MODULE_CONSTANTS = {"REGISTER_MAP", "WHO_AM_I"} +MODULE_CONSTANTS = {"DEVICE_NAME", "REGISTER_MAP", "WHO_AM_I"} @pytest.fixture @@ -31,6 +31,11 @@ def test_returns_device_module(test_module): assert isinstance(test_module, DeviceModule) +def test_device_name_from_schema(test_module): + # Recordings are written under this name, so a reader can match files by it. + assert test_module.DEVICE_NAME == "Tests" + + def test_whoami_defaults_to_zero_when_absent(test_module): # device.yml (application-device metadata) omits whoAmI. assert test_module.WHO_AM_I == 0 @@ -121,13 +126,16 @@ def test_device_register_overrides_core_on_clash(): assert mod.Shadow.address == 0 -def test_headerless_fragment_builds_default_module(): - # A register-only fragment is a valid (nameless) device; name falls back to "Device". +def test_headerless_fragment_builds_nameless_device(): + # A register-only fragment declares no device, so it is nameless and unregistered. + # The empty DEVICE_NAME is what sends a reader to the folder for a prefix, while + # "Device" only keeps the module itself from being anonymous. mod = create_device_module("registers:\n Foo: {address: 40, type: U16, access: Read}\n") - assert mod.__name__ == "Device" + assert mod.DEVICE_NAME == "" assert mod.WHO_AM_I == 0 assert mod.REGISTER_MAP[40].__name__ == "Foo" assert mod.Foo.address == 40 + assert mod.__name__ == "Device" def test_all_covers_declarations_and_module_constants(test_module): diff --git a/tests/device/test_device.py b/tests/device/test_device.py index 570be9c..7d782de 100644 --- a/tests/device/test_device.py +++ b/tests/device/test_device.py @@ -16,6 +16,7 @@ def read(self) -> bytes: def _module(name: str, **attrs: object) -> types.ModuleType: mod = types.ModuleType(name) + mod.DEVICE_NAME = name for key, value in attrs.items(): setattr(mod, key, value) return mod From 37aab41116c72137e37776af9cc8ad716c5002fe Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 16 Aug 2026 09:19:28 +0100 Subject: [PATCH 3/5] Rename the strict flag to require_converters The keyword controlling converter resolution is now require_converters on create_registers, create_device_module and create_dataset_reader. Behavior is unchanged. A custom interfaceType with no matching entry in converters still raises UnknownConverterError, and passing False still decodes it as its native element type. The error message now names the flag rather than explaining what strict meant. --- .../create_device_module.py | 2 +- .../harp-data/src/harp/data/_dataset.py | 9 ++++---- src/packages/harp-device/README.md | 2 +- .../src/harp/device/schema/_emit.py | 22 +++++++++---------- .../src/harp/device/schema/_module.py | 4 ++-- tests/data/test_dataset.py | 11 +++++----- tests/device/test_emit.py | 2 +- 7 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/examples/create_device_module/create_device_module.py b/docs/examples/create_device_module/create_device_module.py index 0896da8..9ec3609 100644 --- a/docs/examples/create_device_module/create_device_module.py +++ b/docs/examples/create_device_module/create_device_module.py @@ -35,7 +35,7 @@ # # behavior = schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()}) # -# An unresolved custom type raises `UnknownConverterError`. Pass `strict=False` to +# An unresolved custom type raises `UnknownConverterError`. Pass `require_converters=False` to # decode it natively instead. A register marked `private` in the schema is emitted # with an underscore-prefixed name. For the parsed schema model rather than a module, # `parse_device_schema(yml_text)` returns that directly. diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index f69db13..83ce9ab 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -230,7 +230,7 @@ def create_dataset_reader( name: str | None = None, resolver: FileNameResolver = default_file_resolver, converters: Mapping[str, Any] | None = None, - strict: bool = True, + require_converters: bool = True, ) -> DatasetReader[DeviceModule]: """Build a :class:`DatasetReader` for a dataset folder, device and all. @@ -242,8 +242,9 @@ def create_dataset_reader( df = reader.read(44) ``schema`` points at the schema file explicitly when it isn't ``root/device.yml``. - ``converters`` and ``strict`` are forwarded to :func:`~harp.device.schema.create_device_module` - for custom ``interfaceType`` decoding; ``name`` and ``resolver`` are forwarded to + ``converters`` and ``require_converters`` are forwarded to + :func:`~harp.device.schema.create_device_module` for custom ``interfaceType`` + decoding; ``name`` and ``resolver`` are forwarded to :class:`DatasetReader`. Use ``DatasetReader(device_module, root)`` directly given a device module already in hand, for example a pre-generated one. """ @@ -255,6 +256,6 @@ def create_dataset_reader( f"or build the device module yourself and use DatasetReader(device_module, root)." ) device_module = create_device_module( - schema_path.read_text(), converters=converters, strict=strict + schema_path.read_text(), converters=converters, require_converters=require_converters ) return DatasetReader(device_module, root_path, name=name, resolver=resolver) diff --git a/src/packages/harp-device/README.md b/src/packages/harp-device/README.md index 5ed1692..3afe707 100644 --- a/src/packages/harp-device/README.md +++ b/src/packages/harp-device/README.md @@ -59,7 +59,7 @@ reg = behavior.REGISTER_MAP[44] # or by address The module is not registered in `sys.modules`, so it has to be bound rather than imported. Names come from the schema at runtime, so they don't autocomplete and aren't statically checked. A generated package on disk gives both. -For a custom `interfaceType`, pass its converter via `converters=`, keyed by `{InterfaceType}Converter` or `{MemberName}Converter`. An unresolved custom type raises `UnknownConverterError`, or pass `strict=False` to decode it natively: +For a custom `interfaceType`, pass its converter via `converters=`, keyed by `{InterfaceType}Converter` or `{MemberName}Converter`. An unresolved custom type raises `UnknownConverterError`, or pass `require_converters=False` to decode it natively: ```python schema.create_device_module(yml_text, converters={"DataConverter": DataConverter()}) diff --git a/src/packages/harp-device/src/harp/device/schema/_emit.py b/src/packages/harp-device/src/harp/device/schema/_emit.py index 9f79e4f..1dfb3eb 100644 --- a/src/packages/harp-device/src/harp/device/schema/_emit.py +++ b/src/packages/harp-device/src/harp/device/schema/_emit.py @@ -242,11 +242,11 @@ def __init__( self, device: Union[DeviceModel, Registers], converters: Optional[Mapping[str, ConverterValue]], - strict: bool, + require_converters: bool, ) -> None: self.device = device self.converters = dict(converters or {}) - self.strict = strict + self.require_converters = require_converters self.group_masks = device.groupMasks or {} self.bit_masks = device.bitMasks or {} self.enums = self._build_enums() @@ -328,12 +328,12 @@ def _extension(self, symbol: str, ctx: ConverterContext) -> Converter[Any]: value = self.converters.get(symbol) if value is not None: return _materialize(value, ctx) - if not self.strict: + if not self.require_converters: return IdentityConverter(ctx.element) raise UnknownConverterError( f"no converter {symbol!r} in converters=; pass " f"converters={{{symbol!r}: Converter>}} " - f"or strict=False to decode as the native type" + f"or require_converters=False to decode as the native type" ) # -- defaults --------------------------------------------------------- @@ -517,7 +517,7 @@ def create_registers( source: str | bytes | DeviceModel | Registers, *, converters: Optional[Mapping[str, ConverterValue]] = None, - strict: bool = True, + require_converters: bool = True, ) -> dict[str, type[RegisterBase[Any]]]: """Emit runtime register classes from a device schema. @@ -530,11 +530,11 @@ def create_registers( :class:`~harp.protocol.Converter` instance or a factory ``(ctx: ConverterContext) -> Converter`` that builds one from the DSL context. A custom type with no matching converter raises - ``UnknownConverterError`` when ``strict`` (the default); ``strict=False`` - decodes it as its native element type instead. A register whose DSL ``visibility`` - is ``private`` is emitted with an underscore-prefixed class (``_Reserved0``), as the - generator emits it. Note that the converter symbol for a payload field derives from - its *verbatim* yml key, not the renamed field. + ``UnknownConverterError``. Pass ``require_converters=False`` to decode it as its + native element type instead. A register whose DSL ``visibility`` is ``private`` is + emitted with an underscore-prefixed class (``_Reserved0``), as the generator emits + it. Note that the converter symbol for a payload field derives from its *verbatim* + yml key, not the renamed field. """ device = source if isinstance(source, Registers) else parse_device_schema(source) - return _Emitter(device, converters, strict).emit() + return _Emitter(device, converters, require_converters).emit() diff --git a/src/packages/harp-device/src/harp/device/schema/_module.py b/src/packages/harp-device/src/harp/device/schema/_module.py index aa7f989..259136f 100644 --- a/src/packages/harp-device/src/harp/device/schema/_module.py +++ b/src/packages/harp-device/src/harp/device/schema/_module.py @@ -64,7 +64,7 @@ def create_device_module( *, name: Optional[str] = None, converters: Optional[Mapping[str, ConverterValue]] = None, - strict: bool = True, + require_converters: bool = True, ) -> DeviceModule: """Emit a module of register classes from ``device.yml`` text. @@ -100,7 +100,7 @@ def create_device_module( behavior.AnalogData """ device = parse_device_schema(text) - emitter = _Emitter(device, converters, strict) + emitter = _Emitter(device, converters, require_converters) registers = emitter.emit() device_name = name or device.device or "" module_name = device_name or _DEFAULT_NAME diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index 0eb58a5..11899c1 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -22,9 +22,9 @@ def _records(cls, n, seed): @pytest.fixture def emitted_module(device_yml): - # strict=False: the test device.yml uses a custom DataConverter we don't inject + # require_converters=False: the test device.yml uses a custom DataConverter we don't inject # here; native decoding is enough to exercise file resolution and parsing. - return create_device_module(device_yml, strict=False) + return create_device_module(device_yml, require_converters=False) @pytest.fixture @@ -295,8 +295,9 @@ def test_reader_derives_name_and_registers_from_module(dataset): def test_create_dataset_reader_builds_module_from_device_yml(dataset, device_yml): mod, _name, root, specs = dataset (root / "device.yml").write_text(device_yml) - # strict=False mirrors the emitted_module fixture (custom DataConverter not injected). - reader = create_dataset_reader(root, strict=False) + # require_converters=False mirrors the emitted_module fixture, which does not + # inject the custom DataConverter either. + reader = create_dataset_reader(root, require_converters=False) assert isinstance(reader, DatasetReader) # Reads match a reader built from an explicitly-generated module. reference = DatasetReader(mod, root) @@ -308,7 +309,7 @@ def test_create_dataset_reader_accepts_explicit_schema_path(dataset, device_yml, _mod, _name, root, specs = dataset schema_path = tmp_path / "elsewhere.yml" # not inside the dataset folder schema_path.write_text(device_yml) - reader = create_dataset_reader(root, schema=schema_path, strict=False) + reader = create_dataset_reader(root, schema=schema_path, require_converters=False) address = next(iter(specs)) assert not reader.read(address).empty diff --git a/tests/device/test_emit.py b/tests/device/test_emit.py index 1a677fb..eebab9a 100644 --- a/tests/device/test_emit.py +++ b/tests/device/test_emit.py @@ -302,7 +302,7 @@ def test_unknown_converter_raises(device_yml): def test_non_strict_falls_back_to_native(device_yml): - regs = create_registers(device_yml, strict=False) + regs = create_registers(device_yml, require_converters=False) # Data decodes as the raw native element (u8[2]) rather than the custom int. reg = regs["CustomMemberConverter"] assert reg.payload_class.payload_dtype.itemsize == 3 From ba3ab72618280edcd12b52c5b143a8108b381a61 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 16 Aug 2026 09:53:21 +0100 Subject: [PATCH 4/5] Validate device identity when reading a dataset DatasetReader compares the WHO_AM_I of its device module with the whoAmI declared by the device.yml in the folder, and raises when they disagree, so a module reused across sessions and reaching the wrong folder fails on construction rather than decoding the files against the wrong register map. The check is skipped when either side declares no identity, and when the schema does not describe a device pyharp can model. A schema that is not well-formed raises instead. validate=False turns off every check the reader performs. --- src/packages/harp-data/README.md | 2 + .../harp-data/src/harp/data/_dataset.py | 43 ++++++++- tests/data/test_dataset.py | 91 ++++++++++++++++++- 3 files changed, 131 insertions(+), 5 deletions(-) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index e1460a3..cf6436f 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -45,6 +45,8 @@ Timestamps are auto-detected per register and placed on the DataFrame index name The `` prefix comes from the `DEVICE_NAME` the device module declares. When there is no declared name, the prefix is read off the folder instead. Pass `name=` to override it, either deliberately or to fix a folder the rule cannot resolve. +When the folder carries a `device.yml` and the module declares an identity, their `whoAmI` values are checked against each other. Reusing a module across sessions and reaching the wrong folder then fails on construction rather than decoding the files against the wrong register map. Pass `validate=False` to turn off every check the reader performs, so a folder whose `device.yml` is damaged can be read with a module obtained elsewhere. + ## Read a single register file `parse_to_dataframe` takes a register and a source, either a path, bytes, or an open binary file, and returns one row per frame: diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 83ce9ab..39264f0 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -6,7 +6,12 @@ from typing import Any, Generic, TypeVar import pandas as pd -from harp.device.schema import DeviceModule, DeviceModuleLike, create_device_module +from harp.device.schema import ( + DeviceModule, + DeviceModuleLike, + create_device_module, + parse_device_schema, +) from harp.protocol import RegisterBase from harp.protocol._constants import _TIMESTAMP_FLAG @@ -60,6 +65,12 @@ class DatasetReader(Generic[M]): the module declares, or read off the folder when it declares none. Pass ``name`` to override that, either deliberately or to read a folder the rule cannot resolve. + When the folder carries a ``device.yml`` and the module declares an identity, their + ``whoAmI`` values are checked against each other. A module paired with the wrong + folder then fails here rather than decoding the files against the wrong register + map. ``validate`` turns off every check the reader performs, so a folder whose + ``device.yml`` is damaged can be read with a module obtained elsewhere. + The reader is typed on the module it was given, so registers stay reachable through :attr:`device_module` at whatever precision that module offers. @@ -76,6 +87,7 @@ def __init__( *, name: str | None = None, resolver: FileNameResolver = default_file_resolver, + validate: bool = True, ) -> None: self._device_module = device_module self._root = Path(root) @@ -83,6 +95,8 @@ def __init__( self._resolver = resolver self._name = self._resolve_name() self._files = dict(self._resolver(self._root, self._name)) + if validate: + self._validate_whoami() @property def root(self) -> Path: @@ -104,6 +118,24 @@ def name(self) -> str: """The ```` prefix used to match binary files.""" return self._name + def _validate_whoami(self) -> None: + expected = getattr(self._device_module, "WHO_AM_I", 0) + if expected == 0: + return + schema_path = self._root / DEVICE_SCHEMA_FILENAME + if not schema_path.is_file(): + return + try: + actual = parse_device_schema(schema_path.read_bytes()).whoAmI + except ValueError: + return + if actual is None or int(actual) == expected: + return + raise ValueError( + f"WhoAmI mismatch: {self._name} expects 0x{expected:04x} but the schema in " + f"{self._root} declares 0x{int(actual):04x}." + ) + def _resolve_name(self) -> str: if self._name_override is not None: return self._name_override @@ -231,6 +263,7 @@ def create_dataset_reader( resolver: FileNameResolver = default_file_resolver, converters: Mapping[str, Any] | None = None, require_converters: bool = True, + validate: bool = True, ) -> DatasetReader[DeviceModule]: """Build a :class:`DatasetReader` for a dataset folder, device and all. @@ -244,9 +277,13 @@ def create_dataset_reader( ``schema`` points at the schema file explicitly when it isn't ``root/device.yml``. ``converters`` and ``require_converters`` are forwarded to :func:`~harp.device.schema.create_device_module` for custom ``interfaceType`` - decoding; ``name`` and ``resolver`` are forwarded to + decoding; ``name``, ``resolver`` and ``validate`` are forwarded to :class:`DatasetReader`. Use ``DatasetReader(device_module, root)`` directly given a device module already in hand, for example a pre-generated one. + + Note ``validate`` cannot rescue a damaged ``device.yml`` here, since the module is + built from that same file and fails before the reader exists. Reading such a folder + means supplying a module obtained elsewhere. """ root_path = Path(root) schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME @@ -258,4 +295,4 @@ def create_dataset_reader( device_module = create_device_module( schema_path.read_text(), converters=converters, require_converters=require_converters ) - return DatasetReader(device_module, root_path, name=name, resolver=resolver) + return DatasetReader(device_module, root_path, name=name, resolver=resolver, validate=validate) diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index 11899c1..b198650 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -168,7 +168,7 @@ def test_name_discovered_without_device_name(dataset): assert DatasetReader(mod, root).name == name -def test_ambiguous_folder_raises(dataset): +def test_ambiguous_folder_raises_until_named(dataset): mod, name, root, specs = dataset del mod.DEVICE_NAME address = next(iter(specs)) @@ -189,7 +189,7 @@ def test_empty_dataset_reads_empty(emitted_module, tmp_path): assert reader.read_all() == {} -def test_unresolvable_prefix_raises(emitted_module, tmp_path): +def test_unresolvable_prefix_raises_on_construction(emitted_module, tmp_path): # Nothing declares a name and nothing on disk suggests one, so no prefix can be # determined. This is the reader failing to be set up, not an empty dataset. del emitted_module.DEVICE_NAME @@ -314,6 +314,93 @@ def test_create_dataset_reader_accepts_explicit_schema_path(dataset, device_yml, assert not reader.read(address).empty +def _with_whoami(device_yml: str, who_am_i: int) -> str: + return f"whoAmI: {who_am_i}\n{device_yml}" + + +def test_whoami_mismatch_raises_on_construction(dataset, device_yml, tmp_path): + # A folder whose schema declares a different device is rejected. + _mod, name, root, specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1216)) + other = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + + elsewhere = tmp_path / "other_session" + elsewhere.mkdir() + for address in specs: + (elsewhere / f"{name}_{address}.bin").write_bytes(b"") + (elsewhere / "device.yml").write_text(_with_whoami(device_yml, 1234)) + + with pytest.raises(ValueError, match="WhoAmI mismatch"): + DatasetReader(other, elsewhere) + + +def test_matching_whoami_does_not_block_read(dataset, device_yml): + _mod, _name, root, specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1216)) + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert not DatasetReader(mod, root).read(next(iter(specs))).empty + + +def test_unregistered_module_skips_whoami_check(dataset, device_yml): + # WHO_AM_I of 0 marks an unregistered device, so there is nothing to check against. + mod, _name, root, _specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1234)) + assert mod.WHO_AM_I == 0 + assert isinstance(DatasetReader(mod, root), DatasetReader) + + +def test_folder_without_schema_skips_whoami_check(dataset, device_yml): + _mod, _name, root, _specs = dataset # no device.yml written into the folder + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert isinstance(DatasetReader(mod, root), DatasetReader) + + +def test_schema_without_whoami_skips_check(dataset, device_yml): + _mod, _name, root, _specs = dataset + (root / "device.yml").write_text(device_yml) # the fixture declares no whoAmI + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert isinstance(DatasetReader(mod, root), DatasetReader) + + +def test_unmodellable_schema_skips_check(dataset, device_yml): + # Well-formed YAML that pyharp cannot describe, such as a newer or older revision, + # must not stop a module that works from decoding the binaries beside it. + _mod, _name, root, specs = dataset + (root / "device.yml").write_text("registers: [this is not a register map]\n") + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert not DatasetReader(mod, root).read(next(iter(specs))).empty + + +def test_validate_false_reads_corrupt_schema(dataset, device_yml): + # The escape hatch: a damaged sidecar must not make a folder unreadable when the + # module decoding it came from elsewhere. + _mod, _name, root, specs = dataset + (root / "device.yml").write_text("registers: {unbalanced\n") + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + reader = DatasetReader(mod, root, validate=False) + assert not reader.read(next(iter(specs))).empty + + +def test_validate_false_skips_mismatch(dataset, device_yml, tmp_path): + _mod, name, root, specs = dataset + (root / "device.yml").write_text(_with_whoami(device_yml, 1234)) + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + assert DatasetReader(mod, root, validate=False).name == name + + +def test_corrupt_schema_is_not_skipped(dataset, device_yml): + # A schema that is not well-formed is a broken dataset rather than one this + # version cannot describe, so it surfaces instead of being skipped. + _mod, _name, root, _specs = dataset + (root / "device.yml").write_text("registers: {unbalanced\n") + mod = create_device_module(_with_whoami(device_yml, 1216), require_converters=False) + with pytest.raises(Exception) as excinfo: + DatasetReader(mod, root) + # Pinning the property rather than the parser: anything deriving from ValueError + # would have been swallowed by the skip, so this must not. + assert not isinstance(excinfo.value, ValueError) + + def test_create_dataset_reader_missing_schema_raises(dataset): _mod, _name, root, _specs = dataset # no device.yml written into the folder with pytest.raises(FileNotFoundError, match="device.yml"): From 64b5a7d7a4c71bf837ca690ed13ba182d573c312 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Sun, 16 Aug 2026 23:02:42 +0100 Subject: [PATCH 5/5] Take the dataset file prefix only from the module DatasetReader no longer infers a device name from the file names in a folder. The prefix comes from name when given and otherwise from the DEVICE_NAME declared by the module, and construction fails when that is empty, as it is for a module built from a schema with no device header. DEVICE_NAME and WHO_AM_I are now read directly rather than through getattr, so a module that does not satisfy DeviceModuleLike raises AttributeError. The dataset fixtures take their prefix from DEVICE_NAME rather than from the module __name__, which were equal only by coincidence, and the chunk test now uses ISO 8601 UTC timestamps in basic format to illustrate lexicographic sorting. --- src/packages/harp-data/README.md | 2 +- .../harp-data/src/harp/data/_dataset.py | 31 +++---- tests/data/test_dataset.py | 80 ++++++------------- 3 files changed, 36 insertions(+), 77 deletions(-) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index cf6436f..6dd4c62 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -43,7 +43,7 @@ df = reader.read(behavior.AnalogData) Timestamps are auto-detected per register and placed on the DataFrame index named `"Time"`: float seconds by default, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. -The `` prefix comes from the `DEVICE_NAME` the device module declares. When there is no declared name, the prefix is read off the folder instead. Pass `name=` to override it, either deliberately or to fix a folder the rule cannot resolve. +The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. When the folder carries a `device.yml` and the module declares an identity, their `whoAmI` values are checked against each other. Reusing a module across sessions and reaching the wrong folder then fails on construction rather than decoding the files against the wrong register map. Pass `validate=False` to turn off every check the reader performs, so a folder whose `device.yml` is damaged can be read with a module obtained elsewhere. diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 39264f0..06f013c 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -27,14 +27,6 @@ """Default filename of the device schema looked up inside a dataset folder.""" -_FILE_STEM = re.compile(r"^(?P.+?)_(?P
\d+)(?:_.*)?$") - - -def _discover_device_names(root: Path) -> set[str]: - stems = (_FILE_STEM.match(path.stem) for path in root.glob("*.bin")) - return {match.group("name") for match in stems if match is not None} - - def default_file_resolver(root: Path, name: str) -> dict[int, list[Path]]: """Harp file format resolver: map address -> sorted ``_
...`` files.""" pattern = re.compile(rf"^{re.escape(name)}_(\d+)(?:_.*)?$") @@ -62,8 +54,8 @@ class DatasetReader(Generic[M]): read on demand. The files are matched by a ```` prefix, taken from the ``DEVICE_NAME`` - the module declares, or read off the folder when it declares none. Pass ``name`` to - override that, either deliberately or to read a folder the rule cannot resolve. + declared by the module. Pass ``name`` to override it, or to supply one when the module + declares an empty name. When the folder carries a ``device.yml`` and the module declares an identity, their ``whoAmI`` values are checked against each other. A module paired with the wrong @@ -119,14 +111,14 @@ def name(self) -> str: return self._name def _validate_whoami(self) -> None: - expected = getattr(self._device_module, "WHO_AM_I", 0) + expected = self._device_module.WHO_AM_I if expected == 0: return - schema_path = self._root / DEVICE_SCHEMA_FILENAME - if not schema_path.is_file(): + path = self._root / DEVICE_SCHEMA_FILENAME + if not path.is_file(): return try: - actual = parse_device_schema(schema_path.read_bytes()).whoAmI + actual = parse_device_schema(path.read_bytes()).whoAmI except ValueError: return if actual is None or int(actual) == expected: @@ -139,17 +131,12 @@ def _validate_whoami(self) -> None: def _resolve_name(self) -> str: if self._name_override is not None: return self._name_override - declared = getattr(self._device_module, "DEVICE_NAME", "") + declared = self._device_module.DEVICE_NAME if declared: return declared - found = _discover_device_names(self._root) - if len(found) == 1: - return found.pop() raise ValueError( - f"The device module declares no DEVICE_NAME, so the file prefix has to come " - f"from {self._root}, which holds " - + (f"data for {sorted(found)}" if found else "no Harp data files") - + ". Pass name= with the prefix to read." + f"The device module declares an empty DEVICE_NAME, so it cannot name the " + f"files under {self._root}. Pass name= with the file prefix to read." ) @property diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index b198650..8b042b1 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -31,7 +31,7 @@ def emitted_module(device_yml): def dataset(emitted_module, tmp_path): """A dataset folder with three app registers; the first is timestamped.""" mod = emitted_module - name = mod.__name__ + name = mod.DEVICE_NAME addresses = [a for a in sorted(mod.REGISTER_MAP) if a >= 32][:3] specs = {} for i, address in enumerate(addresses): @@ -71,7 +71,7 @@ def test_reads_common_registers_not_named_by_module(emitted_module, tmp_path): for cls in (WhoAmI, TimestampSeconds): records = _records(cls, 4, seed=cls.address) buf = bytes(cls.format_bulk(records)) - (tmp_path / f"{mod.__name__}_{cls.address}.bin").write_bytes(buf) + (tmp_path / f"{mod.DEVICE_NAME}_{cls.address}.bin").write_bytes(buf) reader = DatasetReader(mod, tmp_path) # By address, and by the class imported from harp.device, and in read_all. @@ -120,27 +120,32 @@ def test_read_all_keyed_by_register_name(dataset): def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): + # Chunk suffixes in this test are ISO 8601 UTC timestamps in basic format, so + # filename order is chronological order. Written newest first to test the sorting. mod = emitted_module - name = mod.__name__ + name = mod.DEVICE_NAME address = next(a for a in sorted(mod.REGISTER_MAP) if a >= 32) cls = mod.REGISTER_MAP[address] - chunk0 = bytes(cls.format_bulk(_records(cls, 3, seed=1))) - chunk1 = bytes(cls.format_bulk(_records(cls, 2, seed=2))) - (tmp_path / f"{name}_{address}_0.bin").write_bytes(chunk0) - (tmp_path / f"{name}_{address}_1.bin").write_bytes(chunk1) + chunks = { + "20260816T090000Z": bytes(cls.format_bulk(_records(cls, 3, seed=1))), + "20260816T100000Z": bytes(cls.format_bulk(_records(cls, 2, seed=2))), + } + for suffix in reversed(list(chunks)): + (tmp_path / f"{name}_{address}_{suffix}.bin").write_bytes(chunks[suffix]) reader = DatasetReader(mod, tmp_path) - combined = parse_to_dataframe(cls, chunk0 + chunk1, timestamp=False) + combined = parse_to_dataframe(cls, b"".join(chunks.values()), timestamp=False) assert reader.read(cls).reset_index(drop=True).equals(combined) # A specific chunk can still be selected by suffix. - only0 = parse_to_dataframe(cls, chunk0, timestamp=False) - assert reader.read(cls, suffix="0").equals(only0) + earliest = parse_to_dataframe(cls, chunks["20260816T090000Z"], timestamp=False) + assert reader.read(cls, suffix="20260816T090000Z").equals(earliest) def test_non_module_raises_on_register_access(dataset): - _mod, _name, root, _specs = dataset + _mod, name, root, _specs = dataset # Registers are derived lazily; anything without a REGISTER_MAP fails on access. - reader = DatasetReader(object, root) + # name and validate keep construction from reading the module at all. + reader = DatasetReader(object, root, name=name, validate=False) with pytest.raises(AttributeError, match="REGISTER_MAP"): _ = reader.registers @@ -160,27 +165,6 @@ def test_name_from_device_name_not_module_name(dataset): assert DatasetReader(mod, root).name == name -def test_name_discovered_without_device_name(dataset): - # A device package published before DEVICE_NAME existed declares no name, so the - # folder is the only remaining source. - mod, name, root, _specs = dataset - del mod.DEVICE_NAME - assert DatasetReader(mod, root).name == name - - -def test_ambiguous_folder_raises_until_named(dataset): - mod, name, root, specs = dataset - del mod.DEVICE_NAME - address = next(iter(specs)) - (root / f"Other_{address}.bin").write_bytes(b"") - with pytest.raises(ValueError, match="Pass name=") as excinfo: - DatasetReader(mod, root) - # The message names what it found, so the caller knows what to choose between. - assert name in str(excinfo.value) - assert "Other" in str(excinfo.value) - assert DatasetReader(mod, root, name=name).name == name - - def test_empty_dataset_reads_empty(emitted_module, tmp_path): # A session that logged nothing is a dataset with no data, not a failure. reader = DatasetReader(emitted_module, tmp_path) @@ -189,27 +173,15 @@ def test_empty_dataset_reads_empty(emitted_module, tmp_path): assert reader.read_all() == {} -def test_unresolvable_prefix_raises_on_construction(emitted_module, tmp_path): - # Nothing declares a name and nothing on disk suggests one, so no prefix can be - # determined. This is the reader failing to be set up, not an empty dataset. - del emitted_module.DEVICE_NAME +def test_nameless_module_raises_on_construction(dataset): + # A header-less schema declares no device, so its module names nothing and file + # names are not consulted. This is the reader failing to be set up, not empty data. + _mod, name, root, _specs = dataset + nameless = create_device_module("registers:\n Foo: {address: 40, type: U16, access: Read}\n") + assert nameless.DEVICE_NAME == "" with pytest.raises(ValueError, match="Pass name="): - DatasetReader(emitted_module, tmp_path) - - -def test_chunked_files_discover_single_name(emitted_module, tmp_path): - # A stem splits at its first address, so the chunk suffix of a multi-chunk register - # stays out of the device name and one device is not discovered as several. - mod = emitted_module - name = mod.DEVICE_NAME - address = next(a for a in sorted(mod.REGISTER_MAP) if a >= 32) - cls = mod.REGISTER_MAP[address] - for chunk in range(2): - buf = bytes(cls.format_bulk(_records(cls, 2, seed=chunk))) - (tmp_path / f"{name}_{address}_{chunk}.bin").write_bytes(buf) - - del mod.DEVICE_NAME - assert DatasetReader(mod, tmp_path).name == name + DatasetReader(nameless, root) + assert DatasetReader(nameless, root, name=name).name == name def test_missing_register_file_raises(dataset): @@ -262,7 +234,7 @@ def test_files_property_lists_discovered_bins(dataset): def test_read_all_registers_of_mock_device(emitted_module, tmp_path): """Write one .bin per register of the device.yml device, then read them all back.""" mod = emitted_module - name = mod.__name__ + name = mod.DEVICE_NAME expected = {} for address, cls in mod.REGISTER_MAP.items(): records = _records(cls, 4, seed=address)