diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 8c15801..d2d9540 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. +The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when the dataset is opened with an `epoch`. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index a44f2a4..aaa36ad 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -34,9 +34,10 @@ frames = {name: reader.read(name) for name in reader.contents} print(list(frames)) -# Pass an epoch to turn the "Time" index into an absolute `DatetimeIndex` instead -# of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC. -absolute = reader.read(44, epoch=data.REFERENCE_EPOCH) +# Opening with an epoch turns the "Time" index into an absolute `DatetimeIndex` +# instead of float seconds, for every register of the dataset. `REFERENCE_EPOCH` is +# time zero of the Harp clock in UTC. +absolute = data.open_dataset("session.harp", epoch=data.REFERENCE_EPOCH).read(44) print(absolute.index[:3]) # --- Working from a device module already in hand ---------------------------- diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 442f347..7e1db9c 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -53,7 +53,7 @@ df = reader.read(behavior.AnalogData) # by register class Prefer the register class where a generated package supplies one, since it is the only form that type-checks and a misspelling is caught before the folder is read. A module built by `create_device_module` resolves its registers as `Any`, so there the class verifies no more than the name does. -The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Data carrying no timestamp raise unless `timestamp=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. +The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when the dataset is opened with `epoch=REFERENCE_EPOCH`. The anchor is set once for the dataset, since it describes how the recording was made rather than how one register is read. Data carrying no timestamp raise unless `time_index=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. @@ -69,11 +69,11 @@ from my_device import AnalogData df = data.parse_to_dataframe(AnalogData, "AnalogData.bin") df = data.parse_to_dataframe( - AnalogData, raw, timestamp=True, message_type=False, decode_enums=True + AnalogData, raw, time_index=True, epoch=None, keep_type=False, decode_enums=True ) ``` -With `timestamp=True`, the default, the Harp time becomes the DataFrame index named `"Time"`, as float seconds, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is also passed. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. +`time_index` decides the index: `True`, the default, gives the Harp time named `"Time"`, and `False` gives a `RangeIndex`. `epoch` anchors that index, giving float seconds when omitted and an absolute `DatetimeIndex` when set to a datetime such as `REFERENCE_EPOCH`. This function reads one file rather than a dataset, so it takes the anchor directly. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. ## From an already-parsed payload diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 356cd92..16d1335 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -72,6 +72,10 @@ class DatasetReader(Generic[M]): when a register was logged as several ``_
_.bin`` chunks, they are concatenated in filename order. Pass ``resolver`` (a :data:`FileResolver`) to support an alternative on-disk layout. + + ``epoch`` anchors the time index of every read to absolute time, so one dataset is + read on one clock rather than the choice being made per register. It describes how + the recording was anchored. The reference Harp clock starts at :data:`REFERENCE_EPOCH`. """ def __init__( @@ -81,12 +85,14 @@ def __init__( *, name: str | None = None, resolver: FileNameResolver = default_file_resolver, + epoch: datetime | None = None, validate: bool = True, ) -> None: self._device_module = device_module self._root = Path(root) self._name_override = name self._resolver = resolver + self._epoch = epoch self._name = self._resolve_name() self._paths = dict(self._resolver(self._root, self._name)) registers = device_module.REGISTER_MAP @@ -158,9 +164,8 @@ def read( register: RegisterKey, *, suffix: str | None = None, - timestamp: bool = True, - epoch: datetime | None = None, - message_type: bool = False, + time_index: bool = True, + keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, ) -> pd.DataFrame: @@ -182,7 +187,8 @@ def read( ``suffix`` selects a single ``_
_.bin`` chunk, and naming one that is absent raises ``FileNotFoundError`` (default: concatenate every chunk for the address). The remaining options match - :func:`~harp.data.parse_to_dataframe`. + :func:`~harp.data.parse_to_dataframe`, except that the epoch is the one the + reader was opened with. """ cls, address = self._resolve(register) paths = self._resolve_paths(address, suffix) @@ -190,9 +196,9 @@ def read( return parse_to_dataframe( cls, raw, - timestamp=timestamp, - epoch=epoch, - message_type=message_type, + time_index=time_index, + epoch=self._epoch, + keep_type=keep_type, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks, ) @@ -229,6 +235,7 @@ def open_dataset( *, name: str | None = ..., resolver: FileNameResolver = ..., + epoch: datetime | None = ..., validate: bool = ..., ) -> DatasetReader[M]: ... @@ -243,6 +250,7 @@ def open_dataset( resolver: FileNameResolver = ..., converters: Mapping[str, Any] | None = ..., require_converters: bool = ..., + epoch: datetime | None = ..., validate: bool = ..., ) -> DatasetReader[DeviceModule]: ... @@ -256,6 +264,7 @@ def open_dataset( resolver: FileNameResolver = default_file_resolver, converters: Mapping[str, Any] | None = None, require_converters: bool = True, + epoch: datetime | None = None, validate: bool = True, ) -> DatasetReader: """Open a de-multiplexed Harp dataset folder and return a :class:`DatasetReader`. @@ -274,6 +283,10 @@ def open_dataset( decoding. These three parameters describe alternative ways to supply a module, so they are mutually exclusive, and will raise when more than one is specified. + ``epoch`` anchors the time index of every read to absolute time, since the anchor + describes the recording rather than one register. The reference Harp clock starts + at :data:`REFERENCE_EPOCH`, and the default of ``None`` gives float seconds. + ``validate`` cannot rescue a corrupt ``device.yml`` if that schema file is also used to build the module. Reading such a folder always requires supplying a module obtained elsewhere. @@ -286,7 +299,12 @@ def open_dataset( "do not apply when one is given. Drop them, or drop the device module." ) return DatasetReader( - device_module, root_path, name=name, resolver=resolver, validate=validate + device_module, + root_path, + name=name, + resolver=resolver, + epoch=epoch, + validate=validate, ) schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME if not schema_path.is_file(): @@ -302,5 +320,6 @@ def open_dataset( root_path, name=name, resolver=resolver, + epoch=epoch, validate=validate and schema is not None, ) diff --git a/src/packages/harp-data/src/harp/data/_read.py b/src/packages/harp-data/src/harp/data/_read.py index 594b401..8f676c8 100644 --- a/src/packages/harp-data/src/harp/data/_read.py +++ b/src/packages/harp-data/src/harp/data/_read.py @@ -1,3 +1,4 @@ +from datetime import datetime from typing import Any import pandas as pd @@ -76,17 +77,21 @@ def _infer_native_register(raw: bytes) -> type[RegisterBase[Any]]: def read( source: Source, *, - timestamp: bool = True, - message_type: bool = False, + time_index: bool = True, + epoch: datetime | None = None, + keep_type: bool = False, ) -> pd.DataFrame: """Read the binary data of a single register, inferring its native layout. ``source`` may be a file path, raw bytes, or an open binary file. The element type, length and timestamp presence are read from the first frame; values decode to the matching native numpy type (no enum or bit-mask decoding). + The remaining options match :func:`~harp.data.parse_to_dataframe`. """ raw = _read_bytes(source) if len(raw) == 0: return pd.DataFrame() register = _infer_native_register(raw) - return parse_to_dataframe(register, raw, timestamp=timestamp, message_type=message_type) + return parse_to_dataframe( + register, raw, time_index=time_index, epoch=epoch, keep_type=keep_type + ) diff --git a/src/packages/harp-data/src/harp/data/_reader.py b/src/packages/harp-data/src/harp/data/_reader.py index 18bd880..ac6136e 100644 --- a/src/packages/harp-data/src/harp/data/_reader.py +++ b/src/packages/harp-data/src/harp/data/_reader.py @@ -2,14 +2,14 @@ from datetime import datetime from pathlib import Path -from typing import Any, BinaryIO, Union +from typing import Any, BinaryIO import numpy as np import pandas as pd from harp.protocol import RegisterBase from numpy.typing import NDArray -Source = Union[str, Path, bytes, bytearray, memoryview, BinaryIO] +Source = str | Path | bytes | bytearray | memoryview | BinaryIO _MSG_NAMES = np.array(["_NONE", "Read", "Write", "Event"]) @@ -68,38 +68,39 @@ def parse_to_dataframe( register: type[RegisterBase[Any]], source: Source, *, - timestamp: bool = True, - epoch: Union[datetime, None] = None, - message_type: bool = False, + time_index: bool = True, + epoch: datetime | None = None, + keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, ) -> pd.DataFrame: """Parse all frames of ``register`` from ``source`` into a DataFrame. - ``source`` may be a file path, raw bytes, or an open binary file object. When - ``timestamp`` is set, the Harp time becomes the DataFrame index (named - ``"Time"``): float seconds by default, or an absolute ``DatetimeIndex`` when - ``epoch`` is given (e.g. :data:`REFERENCE_EPOCH`). ``message_type`` inserts a - leading column; ``decode_enums`` controls whether enum fields become + ``source`` may be a file path, raw bytes, or an open binary file object. + ``time_index`` makes the Harp time the DataFrame index, named ``"Time"``, and + ``False`` leaves a ``RangeIndex``. ``epoch`` anchors that index to absolute time, + giving a ``DatetimeIndex`` measured from it, where the default of ``None`` gives + float seconds; the Harp clock starts at :data:`REFERENCE_EPOCH`. ``keep_type`` + inserts a leading column; ``decode_enums`` controls whether enum fields become ``pd.Categorical`` (True) or raw codes; ``demux_bit_masks`` expands each flag (``BitMask``) field into one boolean column per flag member (True) or keeps it as a single raw-integer column. """ raw = _read_bytes(source) - _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=timestamp) + _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=time_index) df = payload_to_dataframe(payload, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks) - if message_type and msg_view is not None: + if keep_type and msg_view is not None: df.insert( 0, "message_type", pd.Categorical(_MSG_NAMES[msg_view & 0x03], categories=_MSG_NAMES[1:]), ) - if timestamp: + if time_index: if timestamps is None: if len(df) > 0: raise ValueError( - "Buffer contains no timestamp data; pass timestamp=False to suppress " + "Buffer contains no timestamp data; pass time_index=False to suppress " "the time index." ) seconds = np.empty(0, dtype=np.float64) # empty buffer: empty Time index diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index a33c471..fb44e2b 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -111,7 +111,7 @@ def test_timestamp_false_gives_range_index(dataset): # The diagnostic escape hatch, and the only way to read frames carrying no timestamp. mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - df = reader.read(next(iter(specs)), timestamp=False) + df = reader.read(next(iter(specs)), time_index=False) assert df.index.name is None @@ -129,7 +129,7 @@ def test_untimestamped_frames_raise_value_error(emitted_module, tmp_path): with pytest.raises(ValueError, match="no timestamp data"): reader.read(address) - assert len(reader.read(address, timestamp=False)) == 3 + assert len(reader.read(address, time_index=False)) == 3 def test_time_index_is_float_seconds_without_epoch(dataset): @@ -141,11 +141,11 @@ def test_time_index_is_float_seconds_without_epoch(dataset): assert list(df.index) == [0.0, 1.0, 2.0, 3.0, 4.0] # arange(5) seconds from the fixture -def test_epoch_gives_absolute_datetime_index(dataset): +def test_dataset_epoch_gives_absolute_datetime_index(dataset): mod, _name, root, specs = dataset - reader = DatasetReader(mod, root) + reader = DatasetReader(mod, root, epoch=REFERENCE_EPOCH) address = next(iter(specs)) - df = reader.read(address, epoch=REFERENCE_EPOCH) + df = reader.read(address) assert isinstance(df.index, pd.DatetimeIndex) assert df.index.name == "Time" # Harp seconds are measured from the reference epoch (timestamps were arange(5)). @@ -153,6 +153,24 @@ def test_epoch_gives_absolute_datetime_index(dataset): assert df.index[2] == pd.Timestamp(REFERENCE_EPOCH) + pd.Timedelta(seconds=2) +def test_dataset_epoch_applies_to_every_register(dataset): + # The anchor is set once for the dataset, so no read can index on a different clock + # from its siblings and frames from several registers share one index type. + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root, epoch=REFERENCE_EPOCH) + for address in specs: + assert isinstance(reader.read(address).index, pd.DatetimeIndex) + + +def test_parse_takes_epoch_without_dataset(dataset): + # parse_to_dataframe has no dataset to take the anchor from, so it accepts one. + _mod, _name, _root, specs = dataset + cls, buf = specs[next(iter(specs))] + df = parse_to_dataframe(cls, buf, epoch=REFERENCE_EPOCH) + assert isinstance(df.index, pd.DatetimeIndex) + assert df.index[0] == pd.Timestamp(REFERENCE_EPOCH) + + def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): # Chunk suffixes in this test are ISO 8601 UTC timestamps in basic format, so # filename order is chronological order. Written newest first to test the sorting. diff --git a/tests/device/test_emit.py b/tests/device/test_emit.py index cc1f588..a515cba 100644 --- a/tests/device/test_emit.py +++ b/tests/device/test_emit.py @@ -535,7 +535,7 @@ def test_emitted_register_bulk_matches_oracle(name, device_registers): # Cross-read via harp.data: the shared bytes decode to equal frames through # either class, including column names and decoded enum labels, which now agree. - df_emitted = parse_to_dataframe(emitted, buf, timestamp=False) - df_oracle = parse_to_dataframe(oracle, buf, timestamp=False) + df_emitted = parse_to_dataframe(emitted, buf, time_index=False) + df_oracle = parse_to_dataframe(oracle, buf, time_index=False) assert list(df_emitted.columns) == list(df_oracle.columns) assert df_emitted.equals(df_oracle) diff --git a/tests/protocol/test_register.py b/tests/protocol/test_register.py index e7357d1..be12e48 100644 --- a/tests/protocol/test_register.py +++ b/tests/protocol/test_register.py @@ -237,8 +237,8 @@ def test_empty_buffer_keeps_columns(): # so a buffer carrying no frames still renders all of them. reg = RegisterU32Array(0x28, length=3) records = np.arange(6, dtype=np.uint32).reshape(2, 3) - populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), timestamp=False) - empty = parse_to_dataframe(reg, b"", timestamp=False) + populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), time_index=False) + empty = parse_to_dataframe(reg, b"", time_index=False) assert list(empty.columns) == list(populated.columns) assert empty.dtypes.equals(populated.dtypes) assert len(empty) == 0 @@ -594,7 +594,7 @@ def test_format_bulk_parse_bulk_roundtrip(): reg = RegisterU16(0x20) values = np.array([1, 2, 3], dtype="