From d1d53e97fa2e6c031033c38cfd05b48e2320cf14 Mon Sep 17 00:00:00 2001 From: bcolsen Date: Tue, 9 Apr 2019 17:48:38 -0600 Subject: [PATCH 1/7] biologic file with duplicate columns --- galvani/BioLogic.py | 150 +++++++++++++++++--------------------------- 1 file changed, 58 insertions(+), 92 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index c68769c..3174299 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -144,14 +144,59 @@ VMPmodule_hdr = np.dtype([('shortname', 'S10'), ('version', ' ?? + 9: ('Ece/V', '/mA', '/V', ', I don't see the difference - elif colID in (6, 77): - dtype_dict['Ewe/V'] = ', 8 is either I or ?? - elif colID in (8, 76): - dtype_dict['I/mA'] = ' Date: Thu, 11 Apr 2019 11:59:18 -0600 Subject: [PATCH 2/7] flag fix --- galvani/BioLogic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 3174299..a307f74 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -195,8 +195,9 @@ def VMPdata_dtype_from_colIDs(colIDs): flags2_dict = OrderedDict() for colID in colIDs: if colID in (1, 2, 3, 21, 31, 65): - type_list.append('u1') - field_list.append('flags') + if 'flags' not in field_list: + type_list.append('u1') + field_list.append('flags') if colID == 1: flags_dict['mode'] = (np.uint8(0x03), np.uint8) elif colID == 2: From 56a321f8e383b0c1543aa9a9ac96509d9d17e48c Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Fri, 3 May 2019 18:39:42 +0200 Subject: [PATCH 3/7] Formatting --- galvani/BioLogic.py | 89 +++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 41052f5..9bc9ce4 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -144,48 +144,49 @@ VMPmodule_hdr = np.dtype([('shortname', 'S10'), ('version', ' ?? - 9: ('Ece/V', '/mA', '/V', ' ?? + 9: ('Ece/V', '/mA', '/V', ' Date: Fri, 3 May 2019 18:47:02 +0200 Subject: [PATCH 4/7] Add a VMPdata_colID_flags_map dict --- galvani/BioLogic.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index 9bc9ce4..b79f51d 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -188,6 +188,17 @@ VMPdata_colID_dtype_map = { 480: ('NSR I/%', ' Date: Fri, 3 May 2019 19:12:18 +0200 Subject: [PATCH 5/7] Refactor the VMPdata_dtype_from_colIDs function --- galvani/BioLogic.py | 49 ++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index b79f51d..dbee311 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -9,7 +9,7 @@ import csv from os import SEEK_SET import time from datetime import date, datetime, timedelta -from collections import OrderedDict +from collections import defaultdict, OrderedDict import numpy as np @@ -201,29 +201,46 @@ VMPdata_colID_flag_map = { def VMPdata_dtype_from_colIDs(colIDs): + """Get a numpy record type from a list of column ID numbers. + + The binary layout of the data in the MPR file is described by the sequence + of column ID numbers in the file header. This function converts that + sequence into a numpy dtype which can then be used to load data from the + file with np.frombuffer(). + + Some column IDs refer to small values which are packed into a single byte. + The second return value is a dict describing the bit masks with which to + extract these columns from the flags byte. + + """ type_list = [] - field_list = [] + field_name_counts = defaultdict(int) flags_dict = OrderedDict() flags2_dict = OrderedDict() for colID in colIDs: if colID in VMPdata_colID_flag_map: - if 'flags' not in field_list: - type_list.append('u1') - field_list.append('flags') + # Some column IDs represent boolean flags or small integers + # These are all packed into a single 'flags' byte whose position + # in the overall record is determined by the position of the first + # column ID of flag type. If there are several flags present, + # there is still only one 'flags' int + if 'flags' not in field_name_counts: + type_list.append(('flags', 'u1')) + field_name_counts['flags'] = 1 flag_name, flag_mask, flag_type = VMPdata_colID_flag_map[colID] flags_dict[flag_name] = (np.uint8(flag_mask), flag_type) + elif colID in VMPdata_colID_dtype_map: + field_name, field_type = VMPdata_colID_dtype_map[colID] + field_name_counts[field_name] += 1 + count = field_name_counts[field_name] + if count > 1: + unique_field_name = '%s %d' % (field_name, count) + else: + unique_field_name = field_name + type_list.append((unique_field_name, field_type)) else: - try: - field = VMPdata_colID_dtype_map[colID][0] - if field in field_list: - field += str(len(field_list)) - field_list.append(field) - type_list.append(VMPdata_colID_dtype_map[colID][1]) - except KeyError: - print(list(zip(field_list, type_list))) - raise NotImplementedError("column type %d not implemented" - % colID) - return np.dtype(list(zip(field_list, type_list))), flags_dict, flags2_dict + raise NotImplementedError("column type %d not implemented" % colID) + return np.dtype(type_list), flags_dict, flags2_dict def read_VMP_modules(fileobj, read_module_data=True): From c401aca741ab57a0eae3d9431ad1972b12a2f9a0 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Fri, 3 May 2019 19:15:34 +0200 Subject: [PATCH 6/7] Get rid of flags2_dict as flags2 doesn't actually exist --- galvani/BioLogic.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/galvani/BioLogic.py b/galvani/BioLogic.py index dbee311..e644e68 100644 --- a/galvani/BioLogic.py +++ b/galvani/BioLogic.py @@ -216,7 +216,6 @@ def VMPdata_dtype_from_colIDs(colIDs): type_list = [] field_name_counts = defaultdict(int) flags_dict = OrderedDict() - flags2_dict = OrderedDict() for colID in colIDs: if colID in VMPdata_colID_flag_map: # Some column IDs represent boolean flags or small integers @@ -228,6 +227,9 @@ def VMPdata_dtype_from_colIDs(colIDs): type_list.append(('flags', 'u1')) field_name_counts['flags'] = 1 flag_name, flag_mask, flag_type = VMPdata_colID_flag_map[colID] + # TODO what happens if a flag colID has already been seen + # i.e. if flag_name is already present in flags_dict? + # Does it create a second 'flags' byte in the record? flags_dict[flag_name] = (np.uint8(flag_mask), flag_type) elif colID in VMPdata_colID_dtype_map: field_name, field_type = VMPdata_colID_dtype_map[colID] @@ -240,7 +242,7 @@ def VMPdata_dtype_from_colIDs(colIDs): type_list.append((unique_field_name, field_type)) else: raise NotImplementedError("column type %d not implemented" % colID) - return np.dtype(type_list), flags_dict, flags2_dict + return np.dtype(type_list), flags_dict def read_VMP_modules(fileobj, read_module_data=True): @@ -336,7 +338,7 @@ class MPRfile: else: assert(not any(remaining_headers)) - self.dtype, self.flags_dict, self.flags2_dict = VMPdata_dtype_from_colIDs(column_types) + self.dtype, self.flags_dict = VMPdata_dtype_from_colIDs(column_types) self.data = np.frombuffer(main_data, dtype=self.dtype) assert(self.data.shape[0] == n_data_points) @@ -407,8 +409,5 @@ class MPRfile: if flagname in self.flags_dict: mask, dtype = self.flags_dict[flagname] return np.array(self.data['flags'] & mask, dtype=dtype) - elif flagname in self.flags2_dict: - mask, dtype = self.flags2_dict[flagname] - return np.array(self.data['flags2'] & mask, dtype=dtype) else: raise AttributeError("Flag '%s' not present" % flagname) From 61e2ac8f5778af49126c398dc8f8909594ece70a Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Fri, 3 May 2019 19:46:45 +0200 Subject: [PATCH 7/7] Added unit tests for the VMPdata_dtype_from_colIDs function --- tests/test_BioLogic.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_BioLogic.py b/tests/test_BioLogic.py index f42e85d..c8bf163 100644 --- a/tests/test_BioLogic.py +++ b/tests/test_BioLogic.py @@ -8,7 +8,7 @@ import numpy as np from numpy.testing import assert_array_almost_equal, assert_array_equal import pytest -from galvani import MPTfile, MPRfile +from galvani import BioLogic, MPTfile, MPRfile from galvani.BioLogic import MPTfileCSV, str3 # not exported testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata') @@ -44,6 +44,37 @@ def test_open_MPT_csv_fails_for_bad_file(): MPTfileCSV(os.path.join(testdata_dir, 'bio_logic1.mpr')) +def test_colID_map_uniqueness(): + """Check some uniqueness properties of the VMPdata_colID_xyz maps.""" + field_colIDs = set(BioLogic.VMPdata_colID_dtype_map.keys()) + flag_colIDs = set(BioLogic.VMPdata_colID_flag_map.keys()) + field_names = [v[0] for v in BioLogic.VMPdata_colID_dtype_map.values()] + flag_names = [v[0] for v in BioLogic.VMPdata_colID_flag_map.values()] + assert not field_colIDs.intersection(flag_colIDs) + # 'I/mA' and 'dQ/mA.h' are duplicated + # assert len(set(field_names)) == len(field_names) + assert len(set(flag_names)) == len(flag_names) + assert not set(field_names).intersection(flag_names) + + +@pytest.mark.parametrize('colIDs, expected', [ + ([1, 2, 3], [('flags', 'u1')]), + ([4, 6], [('time/s', '