From 569a5f2a9c5bb5c333beafced4548eb98949935d Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Jun 2025 15:48:19 +0100 Subject: [PATCH 1/7] Add step time/s field for column 182 --- galvani/BioLogic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 4d91549..7c9d3d6 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -316,6 +316,7 @@ VMPdata_colID_dtype_map = { 174: ("/V", " Date: Fri, 13 Jun 2025 15:49:23 +0100 Subject: [PATCH 2/7] Add mode to attempt to read files with unknown columns and only warn --- galvani/BioLogic.py | 77 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 7c9d3d6..aa72045 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -13,9 +13,12 @@ from os import SEEK_SET import time from datetime import date, datetime, timedelta from collections import defaultdict, OrderedDict +import warnings import numpy as np +UNKNOWN_COLUMN_TYPE_HIERARCHY = ("', with an attempt to read + it with a few different dtypes. + + """ self.loop_index = None if isinstance(file_or_path, str): mpr_file = open(file_or_path, "rb") @@ -596,8 +622,41 @@ class MPRfile: assert not any(remaining_headers) - self.dtype, self.flags_dict = VMPdata_dtype_from_colIDs(column_types) - self.data = np.frombuffer(main_data, dtype=self.dtype) + dtypes, self.flags_dict = VMPdata_dtype_from_colIDs( + column_types, error_on_unknown_column=error_on_unknown_column + ) + + unknown_cols = [] + # Iteratively work through the unknown columns and try to read them + if not error_on_unknown_column: + for col, _ in dtypes: + if col.startswith("unknown_colID"): + unknown_cols.append(col) + + if unknown_cols: + # create a list of all possible combinations of dtypes + # for the unknown columns + from itertools import product + perms = product(UNKNOWN_COLUMN_TYPE_HIERARCHY, repeat=len(unknown_cols)) + for perm in perms: + for unknown_col_ind, c in enumerate(unknown_cols): + for ind, (col, _) in enumerate(dtypes): + if c == col: + dtypes[ind] = (col, perm[unknown_col_ind]) + + try: + self.dtype = np.dtype(dtypes) + self.data = np.frombuffer(main_data, dtype=self.dtype) + break + except ValueError: + continue + else: + raise RuntimeError("Unable to read data for unknown columns %s with any of the common dtypes %s", unknown_cols, UNKNOWN_COLUMN_TYPE_HIERARCHY) + + else: + self.dtype = np.dtype(dtypes) + self.data = np.frombuffer(main_data, dtype=self.dtype) + assert self.data.shape[0] == n_data_points # No idea what these 'column types' mean or even if they are actually From 5a207dbf5e248e1d9c12c196c2f1d73bde973bad Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Jun 2025 16:29:21 +0100 Subject: [PATCH 3/7] Add guard for combinatorially exploring more than 3 unknown column data types --- galvani/BioLogic.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index aa72045..428dd9f 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -632,6 +632,8 @@ class MPRfile: for col, _ in dtypes: if col.startswith("unknown_colID"): unknown_cols.append(col) + if len(unknown_cols) > 3: + raise RuntimeError("Too many unknown columns to attempt to read combinatorially: %s" % unknown_cols) if unknown_cols: # create a list of all possible combinations of dtypes From 2c90a2b03860897eb86d292dfee6693cd1cab4e0 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Jun 2025 16:45:58 +0100 Subject: [PATCH 4/7] Temporarily enable the new feature by default --- galvani/BioLogic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 428dd9f..2952b49 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -431,7 +431,7 @@ def parse_BioLogic_date(date_text): return date(tm.tm_year, tm.tm_mon, tm.tm_mday) -def VMPdata_dtype_from_colIDs(colIDs, error_on_unknown_column: bool = True): +def VMPdata_dtype_from_colIDs(colIDs, error_on_unknown_column: bool = False): """Get a numpy record type from a list of column ID numbers. The binary layout of the data in the MPR file is described by the sequence From 30d6098aa076da7ad9b3e7eb7b8de63c1a585b1f Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Jun 2025 18:15:46 +0100 Subject: [PATCH 5/7] Linting --- galvani/BioLogic.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 2952b49..ad2e523 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -633,7 +633,10 @@ class MPRfile: if col.startswith("unknown_colID"): unknown_cols.append(col) if len(unknown_cols) > 3: - raise RuntimeError("Too many unknown columns to attempt to read combinatorially: %s" % unknown_cols) + raise RuntimeError( + "Too many unknown columns to attempt to read combinatorially: %s" + % unknown_cols + ) if unknown_cols: # create a list of all possible combinations of dtypes @@ -653,7 +656,11 @@ class MPRfile: except ValueError: continue else: - raise RuntimeError("Unable to read data for unknown columns %s with any of the common dtypes %s", unknown_cols, UNKNOWN_COLUMN_TYPE_HIERARCHY) + raise RuntimeError( + "Unable to read data for unknown columns %s with any of the common dtypes %s", + unknown_cols, + UNKNOWN_COLUMN_TYPE_HIERARCHY + ) else: self.dtype = np.dtype(dtypes) From a59f263c2bc33de56e053722018683726038360a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Jun 2025 18:23:24 +0100 Subject: [PATCH 6/7] Revert to defaulting to raising an error on unknown cols --- galvani/BioLogic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index ad2e523..0a987db 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -431,7 +431,7 @@ def parse_BioLogic_date(date_text): return date(tm.tm_year, tm.tm_mon, tm.tm_mday) -def VMPdata_dtype_from_colIDs(colIDs, error_on_unknown_column: bool = False): +def VMPdata_dtype_from_colIDs(colIDs, error_on_unknown_column: bool = True): """Get a numpy record type from a list of column ID numbers. The binary layout of the data in the MPR file is described by the sequence From ccaa66b206cd654051d37f25707795d0ca38d534 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Jun 2025 18:24:42 +0100 Subject: [PATCH 7/7] Convert to np.dtype in test --- tests/test_BioLogic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_BioLogic.py b/tests/test_BioLogic.py index 0d8d111..1fb3dc0 100644 --- a/tests/test_BioLogic.py +++ b/tests/test_BioLogic.py @@ -99,7 +99,7 @@ def test_colID_to_dtype(colIDs, expected): return expected_dtype = np.dtype(expected) dtype, flags_dict = BioLogic.VMPdata_dtype_from_colIDs(colIDs) - assert dtype == expected_dtype + assert np.dtype(dtype) == expected_dtype @pytest.mark.parametrize(