From 482a81ec00a1b02a6150850d7ef8194b115b2ea6 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 21 Aug 2026 22:17:12 +0100 Subject: [PATCH 1/3] Rename message_type to keep_type on the reader read, parse_to_dataframe and DatasetReader.read now take keep_type, leaving message_type to to_buffer, to_file and format_bulk, where it carries a MessageType. Passing message_type to a reader now raises instead of accepting any truthy value. The inserted column keeps its name. --- src/packages/harp-data/README.md | 2 +- src/packages/harp-data/src/harp/data/_dataset.py | 4 ++-- src/packages/harp-data/src/harp/data/_read.py | 4 ++-- src/packages/harp-data/src/harp/data/_reader.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 442f347..49a1c1d 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -69,7 +69,7 @@ from my_device import AnalogData df = data.parse_to_dataframe(AnalogData, "AnalogData.bin") df = data.parse_to_dataframe( - AnalogData, raw, timestamp=True, message_type=False, decode_enums=True + AnalogData, raw, timestamp=True, keep_type=False, decode_enums=True ) ``` diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 356cd92..d127a9b 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -160,7 +160,7 @@ def read( suffix: str | None = None, timestamp: bool = True, epoch: datetime | None = None, - message_type: bool = False, + keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, ) -> pd.DataFrame: @@ -192,7 +192,7 @@ def read( raw, timestamp=timestamp, epoch=epoch, - message_type=message_type, + keep_type=keep_type, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks, ) diff --git a/src/packages/harp-data/src/harp/data/_read.py b/src/packages/harp-data/src/harp/data/_read.py index 594b401..99535fe 100644 --- a/src/packages/harp-data/src/harp/data/_read.py +++ b/src/packages/harp-data/src/harp/data/_read.py @@ -77,7 +77,7 @@ def read( source: Source, *, timestamp: bool = True, - message_type: bool = False, + keep_type: bool = False, ) -> pd.DataFrame: """Read the binary data of a single register, inferring its native layout. @@ -89,4 +89,4 @@ def read( if len(raw) == 0: return pd.DataFrame() register = _infer_native_register(raw) - return parse_to_dataframe(register, raw, timestamp=timestamp, message_type=message_type) + return parse_to_dataframe(register, raw, timestamp=timestamp, keep_type=keep_type) diff --git a/src/packages/harp-data/src/harp/data/_reader.py b/src/packages/harp-data/src/harp/data/_reader.py index 18bd880..efb2c97 100644 --- a/src/packages/harp-data/src/harp/data/_reader.py +++ b/src/packages/harp-data/src/harp/data/_reader.py @@ -70,7 +70,7 @@ def parse_to_dataframe( *, timestamp: bool = True, epoch: Union[datetime, None] = None, - message_type: bool = False, + keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, ) -> pd.DataFrame: @@ -79,7 +79,7 @@ def parse_to_dataframe( ``source`` may be a file path, raw bytes, or an open binary file object. When ``timestamp`` is set, the Harp time becomes the DataFrame index (named ``"Time"``): float seconds by default, or an absolute ``DatetimeIndex`` when - ``epoch`` is given (e.g. :data:`REFERENCE_EPOCH`). ``message_type`` inserts a + ``epoch`` is given (e.g. :data:`REFERENCE_EPOCH`). ``keep_type`` inserts a leading column; ``decode_enums`` controls whether enum fields become ``pd.Categorical`` (True) or raw codes; ``demux_bit_masks`` expands each flag (``BitMask``) field into one boolean column per flag member (True) or keeps it @@ -89,7 +89,7 @@ def parse_to_dataframe( _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=timestamp) df = payload_to_dataframe(payload, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks) - if message_type and msg_view is not None: + if keep_type and msg_view is not None: df.insert( 0, "message_type", From b2607bf6ff47b34174a92aff5bc3ea6878092fb9 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Fri, 21 Aug 2026 23:10:32 +0100 Subject: [PATCH 2/3] Replace timestamp and epoch with time_index read, parse_to_dataframe and DatasetReader.read take time_index in place of timestamp and epoch. True indexes by Harp time in float seconds, a datetime such as REFERENCE_EPOCH gives an absolute DatetimeIndex, and False gives a RangeIndex. Anything else raises TypeError, so a value that happens to be truthy no longer passes for a request to index by time. read gains the absolute case, which it had no parameter for before. --- docs/examples/read_dataset/read_dataset.md | 2 +- docs/examples/read_dataset/read_dataset.py | 4 +-- src/packages/harp-data/README.md | 6 ++-- .../harp-data/src/harp/data/_dataset.py | 6 ++-- src/packages/harp-data/src/harp/data/_read.py | 8 +++-- .../harp-data/src/harp/data/_reader.py | 36 ++++++++++++++----- tests/data/test_dataset.py | 16 +++++++-- tests/device/test_emit.py | 4 +-- tests/protocol/test_register.py | 6 ++-- 9 files changed, 58 insertions(+), 30 deletions(-) diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 8c15801..4226eb1 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when an `epoch` is passed. +The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when `time_index` is given a datetime. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index a44f2a4..7dd4b5f 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -34,9 +34,9 @@ frames = {name: reader.read(name) for name in reader.contents} print(list(frames)) -# Pass an epoch to turn the "Time" index into an absolute `DatetimeIndex` instead +# Pass a datetime to turn the "Time" index into an absolute `DatetimeIndex` instead # of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC. -absolute = reader.read(44, epoch=data.REFERENCE_EPOCH) +absolute = reader.read(44, time_index=data.REFERENCE_EPOCH) print(absolute.index[:3]) # --- Working from a device module already in hand ---------------------------- diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 49a1c1d..23b79d0 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -53,7 +53,7 @@ df = reader.read(behavior.AnalogData) # by register class Prefer the register class where a generated package supplies one, since it is the only form that type-checks and a misspelling is caught before the folder is read. A module built by `create_device_module` resolves its registers as `Any`, so there the class verifies no more than the name does. -The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is passed. Data carrying no timestamp raise unless `timestamp=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. +The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when `time_index=REFERENCE_EPOCH` is passed. Data carrying no timestamp raise unless `time_index=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. @@ -69,11 +69,11 @@ from my_device import AnalogData df = data.parse_to_dataframe(AnalogData, "AnalogData.bin") df = data.parse_to_dataframe( - AnalogData, raw, timestamp=True, keep_type=False, decode_enums=True + AnalogData, raw, time_index=True, keep_type=False, decode_enums=True ) ``` -With `timestamp=True`, the default, the Harp time becomes the DataFrame index named `"Time"`, as float seconds, or an absolute `DatetimeIndex` when `epoch=REFERENCE_EPOCH` is also passed. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. +`time_index` decides the index: `True`, the default, gives the Harp time in float seconds named `"Time"`, a datetime such as `REFERENCE_EPOCH` gives an absolute `DatetimeIndex`, and `False` gives a `RangeIndex`. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. ## From an already-parsed payload diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index d127a9b..290a864 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -158,8 +158,7 @@ def read( register: RegisterKey, *, suffix: str | None = None, - timestamp: bool = True, - epoch: datetime | None = None, + time_index: bool | datetime = True, keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, @@ -190,8 +189,7 @@ def read( return parse_to_dataframe( cls, raw, - timestamp=timestamp, - epoch=epoch, + time_index=time_index, keep_type=keep_type, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks, diff --git a/src/packages/harp-data/src/harp/data/_read.py b/src/packages/harp-data/src/harp/data/_read.py index 99535fe..fcc2f4e 100644 --- a/src/packages/harp-data/src/harp/data/_read.py +++ b/src/packages/harp-data/src/harp/data/_read.py @@ -1,4 +1,5 @@ -from typing import Any +from datetime import datetime +from typing import Any, Union import pandas as pd from harp.protocol import ( @@ -76,7 +77,7 @@ def _infer_native_register(raw: bytes) -> type[RegisterBase[Any]]: def read( source: Source, *, - timestamp: bool = True, + time_index: Union[bool, datetime] = True, keep_type: bool = False, ) -> pd.DataFrame: """Read the binary data of a single register, inferring its native layout. @@ -84,9 +85,10 @@ def read( ``source`` may be a file path, raw bytes, or an open binary file. The element type, length and timestamp presence are read from the first frame; values decode to the matching native numpy type (no enum or bit-mask decoding). + ``time_index`` matches :func:`~harp.data.parse_to_dataframe`. """ raw = _read_bytes(source) if len(raw) == 0: return pd.DataFrame() register = _infer_native_register(raw) - return parse_to_dataframe(register, raw, timestamp=timestamp, keep_type=keep_type) + return parse_to_dataframe(register, raw, time_index=time_index, keep_type=keep_type) diff --git a/src/packages/harp-data/src/harp/data/_reader.py b/src/packages/harp-data/src/harp/data/_reader.py index efb2c97..54c7967 100644 --- a/src/packages/harp-data/src/harp/data/_reader.py +++ b/src/packages/harp-data/src/harp/data/_reader.py @@ -28,6 +28,22 @@ def _time_index(seconds: NDArray[np.float64], epoch: datetime | None) -> pd.Inde ) +def _resolve_epoch(time_index: object) -> Union[datetime, None]: + """The epoch a ``time_index`` argument selects, rejecting anything else. + + Only ``bool`` and ``datetime`` are accepted, so a value that happens to be truthy + cannot pass for a request to index by time. + """ + if isinstance(time_index, bool): + return None + if isinstance(time_index, datetime): + return time_index + raise TypeError( + f"time_index must be a bool or a datetime such as REFERENCE_EPOCH, " + f"not {type(time_index).__name__}." + ) + + def _read_bytes(source: Source) -> bytes: if isinstance(source, (bytes, bytearray, memoryview)): return bytes(source) @@ -68,25 +84,27 @@ def parse_to_dataframe( register: type[RegisterBase[Any]], source: Source, *, - timestamp: bool = True, - epoch: Union[datetime, None] = None, + time_index: Union[bool, datetime] = True, keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, ) -> pd.DataFrame: """Parse all frames of ``register`` from ``source`` into a DataFrame. - ``source`` may be a file path, raw bytes, or an open binary file object. When - ``timestamp`` is set, the Harp time becomes the DataFrame index (named - ``"Time"``): float seconds by default, or an absolute ``DatetimeIndex`` when - ``epoch`` is given (e.g. :data:`REFERENCE_EPOCH`). ``keep_type`` inserts a + ``source`` may be a file path, raw bytes, or an open binary file object. + ``time_index`` makes the Harp time the DataFrame index, named ``"Time"``: ``True`` + for float seconds, a ``datetime`` such as :data:`REFERENCE_EPOCH` for an absolute + ``DatetimeIndex``, and ``False`` for a ``RangeIndex``. ``keep_type`` inserts a leading column; ``decode_enums`` controls whether enum fields become ``pd.Categorical`` (True) or raw codes; ``demux_bit_masks`` expands each flag (``BitMask``) field into one boolean column per flag member (True) or keeps it as a single raw-integer column. """ + epoch = _resolve_epoch(time_index) raw = _read_bytes(source) - _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=timestamp) + _data, timestamps, msg_view, payload = register.parse_bulk( + raw, parse_timestamp=time_index is not False + ) df = payload_to_dataframe(payload, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks) if keep_type and msg_view is not None: @@ -95,11 +113,11 @@ def parse_to_dataframe( "message_type", pd.Categorical(_MSG_NAMES[msg_view & 0x03], categories=_MSG_NAMES[1:]), ) - if timestamp: + if time_index is not False: if timestamps is None: if len(df) > 0: raise ValueError( - "Buffer contains no timestamp data; pass timestamp=False to suppress " + "Buffer contains no timestamp data; pass time_index=False to suppress " "the time index." ) seconds = np.empty(0, dtype=np.float64) # empty buffer: empty Time index diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index a33c471..5e10ec6 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -111,7 +111,7 @@ def test_timestamp_false_gives_range_index(dataset): # The diagnostic escape hatch, and the only way to read frames carrying no timestamp. mod, _name, root, specs = dataset reader = DatasetReader(mod, root) - df = reader.read(next(iter(specs)), timestamp=False) + df = reader.read(next(iter(specs)), time_index=False) assert df.index.name is None @@ -129,7 +129,7 @@ def test_untimestamped_frames_raise_value_error(emitted_module, tmp_path): with pytest.raises(ValueError, match="no timestamp data"): reader.read(address) - assert len(reader.read(address, timestamp=False)) == 3 + assert len(reader.read(address, time_index=False)) == 3 def test_time_index_is_float_seconds_without_epoch(dataset): @@ -145,7 +145,7 @@ def test_epoch_gives_absolute_datetime_index(dataset): mod, _name, root, specs = dataset reader = DatasetReader(mod, root) address = next(iter(specs)) - df = reader.read(address, epoch=REFERENCE_EPOCH) + df = reader.read(address, time_index=REFERENCE_EPOCH) assert isinstance(df.index, pd.DatetimeIndex) assert df.index.name == "Time" # Harp seconds are measured from the reference epoch (timestamps were arange(5)). @@ -153,6 +153,16 @@ def test_epoch_gives_absolute_datetime_index(dataset): assert df.index[2] == pd.Timestamp(REFERENCE_EPOCH) + pd.Timedelta(seconds=2) +def test_time_index_raises_type_error_for_other_types(dataset): + # Only a bool or a datetime selects the index, so a value that merely happens to be + # truthy cannot pass for a request to index by time. + mod, _name, root, specs = dataset + reader = DatasetReader(mod, root) + address = next(iter(specs)) + with pytest.raises(TypeError, match="time_index"): + reader.read(address, time_index="Time") # type: ignore[arg-type] + + def test_suffix_chunks_are_concatenated(emitted_module, tmp_path): # Chunk suffixes in this test are ISO 8601 UTC timestamps in basic format, so # filename order is chronological order. Written newest first to test the sorting. diff --git a/tests/device/test_emit.py b/tests/device/test_emit.py index cc1f588..a515cba 100644 --- a/tests/device/test_emit.py +++ b/tests/device/test_emit.py @@ -535,7 +535,7 @@ def test_emitted_register_bulk_matches_oracle(name, device_registers): # Cross-read via harp.data: the shared bytes decode to equal frames through # either class, including column names and decoded enum labels, which now agree. - df_emitted = parse_to_dataframe(emitted, buf, timestamp=False) - df_oracle = parse_to_dataframe(oracle, buf, timestamp=False) + df_emitted = parse_to_dataframe(emitted, buf, time_index=False) + df_oracle = parse_to_dataframe(oracle, buf, time_index=False) assert list(df_emitted.columns) == list(df_oracle.columns) assert df_emitted.equals(df_oracle) diff --git a/tests/protocol/test_register.py b/tests/protocol/test_register.py index e7357d1..be12e48 100644 --- a/tests/protocol/test_register.py +++ b/tests/protocol/test_register.py @@ -237,8 +237,8 @@ def test_empty_buffer_keeps_columns(): # so a buffer carrying no frames still renders all of them. reg = RegisterU32Array(0x28, length=3) records = np.arange(6, dtype=np.uint32).reshape(2, 3) - populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), timestamp=False) - empty = parse_to_dataframe(reg, b"", timestamp=False) + populated = parse_to_dataframe(reg, bytes(reg.format_bulk(records)), time_index=False) + empty = parse_to_dataframe(reg, b"", time_index=False) assert list(empty.columns) == list(populated.columns) assert empty.dtypes.equals(populated.dtypes) assert len(empty) == 0 @@ -594,7 +594,7 @@ def test_format_bulk_parse_bulk_roundtrip(): reg = RegisterU16(0x20) values = np.array([1, 2, 3], dtype=" Date: Sat, 22 Aug 2026 01:24:06 +0100 Subject: [PATCH 3/3] Split the epoch out of time_index time_index narrows to a bool, and the epoch it used to carry becomes its own parameter. open_dataset and the DatasetReader constructor take the epoch for the whole dataset, so every register reads on one clock and DatasetReader.read no longer sets one at all. parse_to_dataframe and read keep an epoch parameter, having no dataset to take it from. _resolve_epoch goes with the union, and with it the TypeError it raised for a value that was neither a bool nor a datetime, since epoch is now the parameter that takes one. The three remaining Union annotations in harp-data become pipe unions, so the package now spells every union the same way. --- docs/examples/read_dataset/read_dataset.md | 2 +- docs/examples/read_dataset/read_dataset.py | 7 ++-- src/packages/harp-data/README.md | 6 +-- .../harp-data/src/harp/data/_dataset.py | 27 +++++++++++-- src/packages/harp-data/src/harp/data/_read.py | 11 ++++-- .../harp-data/src/harp/data/_reader.py | 39 ++++++------------- tests/data/test_dataset.py | 28 ++++++++----- 7 files changed, 68 insertions(+), 52 deletions(-) diff --git a/docs/examples/read_dataset/read_dataset.md b/docs/examples/read_dataset/read_dataset.md index 4226eb1..d2d9540 100644 --- a/docs/examples/read_dataset/read_dataset.md +++ b/docs/examples/read_dataset/read_dataset.md @@ -4,7 +4,7 @@ A Harp acquisition is usually saved as a **de-multiplexed dataset folder**: one This is the recommended entry point for a recorded session on disk. To decode a single loose `.bin` file instead, see [Reading Data into a DataFrame](../read_data_to_dataframe/read_data_to_dataframe.md). -The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when `time_index` is given a datetime. +The quickest way in is `open_dataset(folder)`. It finds the `device.yml` inside the folder, builds the device module, and returns a reader ready to go. Given a device module already in hand, for example from a pre-generated package, pass it as the second argument, `open_dataset(folder, module)`. A register is then read by class, by name, or by address. The Harp time becomes the `"Time"` index, as float seconds or an absolute `DatetimeIndex` when the dataset is opened with an `epoch`. ```python diff --git a/docs/examples/read_dataset/read_dataset.py b/docs/examples/read_dataset/read_dataset.py index 7dd4b5f..aaa36ad 100644 --- a/docs/examples/read_dataset/read_dataset.py +++ b/docs/examples/read_dataset/read_dataset.py @@ -34,9 +34,10 @@ frames = {name: reader.read(name) for name in reader.contents} print(list(frames)) -# Pass a datetime to turn the "Time" index into an absolute `DatetimeIndex` instead -# of float seconds. `REFERENCE_EPOCH` is time zero of the Harp clock in UTC. -absolute = reader.read(44, time_index=data.REFERENCE_EPOCH) +# Opening with an epoch turns the "Time" index into an absolute `DatetimeIndex` +# instead of float seconds, for every register of the dataset. `REFERENCE_EPOCH` is +# time zero of the Harp clock in UTC. +absolute = data.open_dataset("session.harp", epoch=data.REFERENCE_EPOCH).read(44) print(absolute.index[:3]) # --- Working from a device module already in hand ---------------------------- diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index 23b79d0..7e1db9c 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -53,7 +53,7 @@ df = reader.read(behavior.AnalogData) # by register class Prefer the register class where a generated package supplies one, since it is the only form that type-checks and a misspelling is caught before the folder is read. A module built by `create_device_module` resolves its registers as `Any`, so there the class verifies no more than the name does. -The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when `time_index=REFERENCE_EPOCH` is passed. Data carrying no timestamp raise unless `time_index=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. +The Harp time becomes the DataFrame index named `"Time"`, as float seconds by default or an absolute `DatetimeIndex` when the dataset is opened with `epoch=REFERENCE_EPOCH`. The anchor is set once for the dataset, since it describes how the recording was made rather than how one register is read. Data carrying no timestamp raise unless `time_index=False` is passed. Multi-chunk registers logged as `_
_.bin` are concatenated in filename order; pass a `resolver` to support an alternative on-disk layout. `paths` reports what the resolver found, keyed by address, which is where a custom layout or a chunked register can be checked. The `` prefix comes from the `DEVICE_NAME` declared by the device module. Pass `name=` to override it, or to supply one when the module declares an empty name. @@ -69,11 +69,11 @@ from my_device import AnalogData df = data.parse_to_dataframe(AnalogData, "AnalogData.bin") df = data.parse_to_dataframe( - AnalogData, raw, time_index=True, keep_type=False, decode_enums=True + AnalogData, raw, time_index=True, epoch=None, keep_type=False, decode_enums=True ) ``` -`time_index` decides the index: `True`, the default, gives the Harp time in float seconds named `"Time"`, a datetime such as `REFERENCE_EPOCH` gives an absolute `DatetimeIndex`, and `False` gives a `RangeIndex`. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. +`time_index` decides the index: `True`, the default, gives the Harp time named `"Time"`, and `False` gives a `RangeIndex`. `epoch` anchors that index, giving float seconds when omitted and an absolute `DatetimeIndex` when set to a datetime such as `REFERENCE_EPOCH`. This function reads one file rather than a dataset, so it takes the anchor directly. Enum fields decode to `pd.Categorical`, and `decode_enums=False` keeps raw codes. ## From an already-parsed payload diff --git a/src/packages/harp-data/src/harp/data/_dataset.py b/src/packages/harp-data/src/harp/data/_dataset.py index 290a864..16d1335 100644 --- a/src/packages/harp-data/src/harp/data/_dataset.py +++ b/src/packages/harp-data/src/harp/data/_dataset.py @@ -72,6 +72,10 @@ class DatasetReader(Generic[M]): when a register was logged as several ``_
_.bin`` chunks, they are concatenated in filename order. Pass ``resolver`` (a :data:`FileResolver`) to support an alternative on-disk layout. + + ``epoch`` anchors the time index of every read to absolute time, so one dataset is + read on one clock rather than the choice being made per register. It describes how + the recording was anchored. The reference Harp clock starts at :data:`REFERENCE_EPOCH`. """ def __init__( @@ -81,12 +85,14 @@ def __init__( *, name: str | None = None, resolver: FileNameResolver = default_file_resolver, + epoch: datetime | None = None, validate: bool = True, ) -> None: self._device_module = device_module self._root = Path(root) self._name_override = name self._resolver = resolver + self._epoch = epoch self._name = self._resolve_name() self._paths = dict(self._resolver(self._root, self._name)) registers = device_module.REGISTER_MAP @@ -158,7 +164,7 @@ def read( register: RegisterKey, *, suffix: str | None = None, - time_index: bool | datetime = True, + time_index: bool = True, keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, @@ -181,7 +187,8 @@ def read( ``suffix`` selects a single ``_
_.bin`` chunk, and naming one that is absent raises ``FileNotFoundError`` (default: concatenate every chunk for the address). The remaining options match - :func:`~harp.data.parse_to_dataframe`. + :func:`~harp.data.parse_to_dataframe`, except that the epoch is the one the + reader was opened with. """ cls, address = self._resolve(register) paths = self._resolve_paths(address, suffix) @@ -190,6 +197,7 @@ def read( cls, raw, time_index=time_index, + epoch=self._epoch, keep_type=keep_type, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks, @@ -227,6 +235,7 @@ def open_dataset( *, name: str | None = ..., resolver: FileNameResolver = ..., + epoch: datetime | None = ..., validate: bool = ..., ) -> DatasetReader[M]: ... @@ -241,6 +250,7 @@ def open_dataset( resolver: FileNameResolver = ..., converters: Mapping[str, Any] | None = ..., require_converters: bool = ..., + epoch: datetime | None = ..., validate: bool = ..., ) -> DatasetReader[DeviceModule]: ... @@ -254,6 +264,7 @@ def open_dataset( resolver: FileNameResolver = default_file_resolver, converters: Mapping[str, Any] | None = None, require_converters: bool = True, + epoch: datetime | None = None, validate: bool = True, ) -> DatasetReader: """Open a de-multiplexed Harp dataset folder and return a :class:`DatasetReader`. @@ -272,6 +283,10 @@ def open_dataset( decoding. These three parameters describe alternative ways to supply a module, so they are mutually exclusive, and will raise when more than one is specified. + ``epoch`` anchors the time index of every read to absolute time, since the anchor + describes the recording rather than one register. The reference Harp clock starts + at :data:`REFERENCE_EPOCH`, and the default of ``None`` gives float seconds. + ``validate`` cannot rescue a corrupt ``device.yml`` if that schema file is also used to build the module. Reading such a folder always requires supplying a module obtained elsewhere. @@ -284,7 +299,12 @@ def open_dataset( "do not apply when one is given. Drop them, or drop the device module." ) return DatasetReader( - device_module, root_path, name=name, resolver=resolver, validate=validate + device_module, + root_path, + name=name, + resolver=resolver, + epoch=epoch, + validate=validate, ) schema_path = Path(schema) if schema is not None else root_path / DEVICE_SCHEMA_FILENAME if not schema_path.is_file(): @@ -300,5 +320,6 @@ def open_dataset( root_path, name=name, resolver=resolver, + epoch=epoch, validate=validate and schema is not None, ) diff --git a/src/packages/harp-data/src/harp/data/_read.py b/src/packages/harp-data/src/harp/data/_read.py index fcc2f4e..8f676c8 100644 --- a/src/packages/harp-data/src/harp/data/_read.py +++ b/src/packages/harp-data/src/harp/data/_read.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Union +from typing import Any import pandas as pd from harp.protocol import ( @@ -77,7 +77,8 @@ def _infer_native_register(raw: bytes) -> type[RegisterBase[Any]]: def read( source: Source, *, - time_index: Union[bool, datetime] = True, + time_index: bool = True, + epoch: datetime | None = None, keep_type: bool = False, ) -> pd.DataFrame: """Read the binary data of a single register, inferring its native layout. @@ -85,10 +86,12 @@ def read( ``source`` may be a file path, raw bytes, or an open binary file. The element type, length and timestamp presence are read from the first frame; values decode to the matching native numpy type (no enum or bit-mask decoding). - ``time_index`` matches :func:`~harp.data.parse_to_dataframe`. + The remaining options match :func:`~harp.data.parse_to_dataframe`. """ raw = _read_bytes(source) if len(raw) == 0: return pd.DataFrame() register = _infer_native_register(raw) - return parse_to_dataframe(register, raw, time_index=time_index, keep_type=keep_type) + return parse_to_dataframe( + register, raw, time_index=time_index, epoch=epoch, keep_type=keep_type + ) diff --git a/src/packages/harp-data/src/harp/data/_reader.py b/src/packages/harp-data/src/harp/data/_reader.py index 54c7967..ac6136e 100644 --- a/src/packages/harp-data/src/harp/data/_reader.py +++ b/src/packages/harp-data/src/harp/data/_reader.py @@ -2,14 +2,14 @@ from datetime import datetime from pathlib import Path -from typing import Any, BinaryIO, Union +from typing import Any, BinaryIO import numpy as np import pandas as pd from harp.protocol import RegisterBase from numpy.typing import NDArray -Source = Union[str, Path, bytes, bytearray, memoryview, BinaryIO] +Source = str | Path | bytes | bytearray | memoryview | BinaryIO _MSG_NAMES = np.array(["_NONE", "Read", "Write", "Event"]) @@ -28,22 +28,6 @@ def _time_index(seconds: NDArray[np.float64], epoch: datetime | None) -> pd.Inde ) -def _resolve_epoch(time_index: object) -> Union[datetime, None]: - """The epoch a ``time_index`` argument selects, rejecting anything else. - - Only ``bool`` and ``datetime`` are accepted, so a value that happens to be truthy - cannot pass for a request to index by time. - """ - if isinstance(time_index, bool): - return None - if isinstance(time_index, datetime): - return time_index - raise TypeError( - f"time_index must be a bool or a datetime such as REFERENCE_EPOCH, " - f"not {type(time_index).__name__}." - ) - - def _read_bytes(source: Source) -> bytes: if isinstance(source, (bytes, bytearray, memoryview)): return bytes(source) @@ -84,7 +68,8 @@ def parse_to_dataframe( register: type[RegisterBase[Any]], source: Source, *, - time_index: Union[bool, datetime] = True, + time_index: bool = True, + epoch: datetime | None = None, keep_type: bool = False, decode_enums: bool = True, demux_bit_masks: bool = False, @@ -92,19 +77,17 @@ def parse_to_dataframe( """Parse all frames of ``register`` from ``source`` into a DataFrame. ``source`` may be a file path, raw bytes, or an open binary file object. - ``time_index`` makes the Harp time the DataFrame index, named ``"Time"``: ``True`` - for float seconds, a ``datetime`` such as :data:`REFERENCE_EPOCH` for an absolute - ``DatetimeIndex``, and ``False`` for a ``RangeIndex``. ``keep_type`` inserts a - leading column; ``decode_enums`` controls whether enum fields become + ``time_index`` makes the Harp time the DataFrame index, named ``"Time"``, and + ``False`` leaves a ``RangeIndex``. ``epoch`` anchors that index to absolute time, + giving a ``DatetimeIndex`` measured from it, where the default of ``None`` gives + float seconds; the Harp clock starts at :data:`REFERENCE_EPOCH`. ``keep_type`` + inserts a leading column; ``decode_enums`` controls whether enum fields become ``pd.Categorical`` (True) or raw codes; ``demux_bit_masks`` expands each flag (``BitMask``) field into one boolean column per flag member (True) or keeps it as a single raw-integer column. """ - epoch = _resolve_epoch(time_index) raw = _read_bytes(source) - _data, timestamps, msg_view, payload = register.parse_bulk( - raw, parse_timestamp=time_index is not False - ) + _data, timestamps, msg_view, payload = register.parse_bulk(raw, parse_timestamp=time_index) df = payload_to_dataframe(payload, decode_enums=decode_enums, demux_bit_masks=demux_bit_masks) if keep_type and msg_view is not None: @@ -113,7 +96,7 @@ def parse_to_dataframe( "message_type", pd.Categorical(_MSG_NAMES[msg_view & 0x03], categories=_MSG_NAMES[1:]), ) - if time_index is not False: + if time_index: if timestamps is None: if len(df) > 0: raise ValueError( diff --git a/tests/data/test_dataset.py b/tests/data/test_dataset.py index 5e10ec6..fb44e2b 100644 --- a/tests/data/test_dataset.py +++ b/tests/data/test_dataset.py @@ -141,11 +141,11 @@ def test_time_index_is_float_seconds_without_epoch(dataset): assert list(df.index) == [0.0, 1.0, 2.0, 3.0, 4.0] # arange(5) seconds from the fixture -def test_epoch_gives_absolute_datetime_index(dataset): +def test_dataset_epoch_gives_absolute_datetime_index(dataset): mod, _name, root, specs = dataset - reader = DatasetReader(mod, root) + reader = DatasetReader(mod, root, epoch=REFERENCE_EPOCH) address = next(iter(specs)) - df = reader.read(address, time_index=REFERENCE_EPOCH) + df = reader.read(address) assert isinstance(df.index, pd.DatetimeIndex) assert df.index.name == "Time" # Harp seconds are measured from the reference epoch (timestamps were arange(5)). @@ -153,14 +153,22 @@ def test_epoch_gives_absolute_datetime_index(dataset): assert df.index[2] == pd.Timestamp(REFERENCE_EPOCH) + pd.Timedelta(seconds=2) -def test_time_index_raises_type_error_for_other_types(dataset): - # Only a bool or a datetime selects the index, so a value that merely happens to be - # truthy cannot pass for a request to index by time. +def test_dataset_epoch_applies_to_every_register(dataset): + # The anchor is set once for the dataset, so no read can index on a different clock + # from its siblings and frames from several registers share one index type. mod, _name, root, specs = dataset - reader = DatasetReader(mod, root) - address = next(iter(specs)) - with pytest.raises(TypeError, match="time_index"): - reader.read(address, time_index="Time") # type: ignore[arg-type] + reader = DatasetReader(mod, root, epoch=REFERENCE_EPOCH) + for address in specs: + assert isinstance(reader.read(address).index, pd.DatetimeIndex) + + +def test_parse_takes_epoch_without_dataset(dataset): + # parse_to_dataframe has no dataset to take the anchor from, so it accepts one. + _mod, _name, _root, specs = dataset + cls, buf = specs[next(iter(specs))] + df = parse_to_dataframe(cls, buf, epoch=REFERENCE_EPOCH) + assert isinstance(df.index, pd.DatetimeIndex) + assert df.index[0] == pd.Timestamp(REFERENCE_EPOCH) def test_suffix_chunks_are_concatenated(emitted_module, tmp_path):