From a0700b027685b28e1e1978765cac43bab21d8b4c Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sun, 11 Oct 2020 16:33:25 +0300 Subject: [PATCH 1/8] Add extra columns to Global and Battery Data tables for v5.26 Arbin software v5.26 adds additional columns to these existing tables; these need to be added to the output database so that the conversion does not fail. --- galvani/res2sqlite.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/galvani/res2sqlite.py b/galvani/res2sqlite.py index eff978a..f5e5420 100755 --- a/galvani/res2sqlite.py +++ b/galvani/res2sqlite.py @@ -56,8 +56,17 @@ CREATE TABLE Global_Table Log_Aux_Data_Flag INTEGER, Log_Event_Data_Flag INTEGER, Log_Smart_Battery_Data_Flag INTEGER, + -- The following items are in 5.26 but not in 5.23 + Log_Can_BMS_Data_Flag INTEGER DEFAULT NULL, + Software_Version TEXT DEFAULT NULL, + Serial_Number TEXT DEFAULT NULL, + Schedule_Version TEXT DEFAULT NULL, + MASS REAL DEFAULT NULL, + Specific_Capacity REAL DEFAULT NULL, + Capacity REAL DEFAULT NULL, + -- Item_ID exists in all versions Item_ID TEXT, - -- Version 1.14 ends here, version 5.23 continues + -- These items are in 5.26 and 5.23 but not in 1.14 Mapped_Aux_Conc_CNumber INTEGER DEFAULT NULL, Mapped_Aux_DI_CNumber INTEGER DEFAULT NULL, Mapped_Aux_DO_CNumber INTEGER DEFAULT NULL @@ -222,6 +231,10 @@ CREATE TABLE Smart_Battery_Data_Table ChargingCurrent REAL DEFAULT NULL, ChargingVoltage REAL DEFAULT NULL, ManufacturerData REAL DEFAULT NULL, + -- Version 5.23 ends here, version 5.26 continues + BATMAN_Status INTEGER DEFAULT NULL, + DTM_PDM_Status INTEGER DEFAULT NULL, + FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) ); """, From 60639299b828487051d885c010e7ab951f2f3646 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sun, 11 Oct 2020 20:40:23 +0300 Subject: [PATCH 2/8] Parse out version number from Version_Table Previously the version was compared for strict equality so a higher version did not match. --- galvani/res2sqlite.py | 46 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/galvani/res2sqlite.py b/galvani/res2sqlite.py index f5e5420..1899eaa 100755 --- a/galvani/res2sqlite.py +++ b/galvani/res2sqlite.py @@ -450,11 +450,51 @@ def mdb_get_data(s3db, filename, table): raise ValueError("'%s' is in neither mdb_tables_text nor mdb_tables_numeric" % table) +def mdb_get_version(filename): + """Get the version number from an Arbin .res file. + + Reads the Version_Table and parses the version from Version_Schema_Field. + """ + print("Reading version number...") + try: + with sp.Popen(['mdb-export', filename, 'Version_Table'], + bufsize=-1, stdin=sp.DEVNULL, stdout=sp.PIPE, + universal_newlines=True) as mdb_sql: + mdb_csv = csv.reader(mdb_sql.stdout) + mdb_headers = next(mdb_csv) + mdb_values = next(mdb_csv) + try: + next(mdb_csv) + except StopIteration: + pass + else: + raise ValueError('Version_Table of %s lists multiple versions' % filename) + except OSError as e: + if e.errno == 2: + raise RuntimeError('Could not locate the `mdb-export` executable. ' + 'Check that mdbtools is properly installed.') + else: + raise + if 'Version_Schema_Field' not in mdb_headers: + raise ValueError('Version_Table of %s does not contain a Version_Schema_Field column' + % filename) + version_fields = dict(zip(mdb_headers, mdb_values)) + version_text = version_fields['Version_Schema_Field'] + version_match = re.fullmatch('Results File ([.0-9]+)', version_text) + if not version_match: + raise ValueError('File version "%s" did not match expected format' % version_text) + version_string = version_match.group(1) + version_tuple = tuple(map(int, version_string.split('.'))) + return version_tuple + + def convert_arbin_to_sqlite(input_file, output_file): """Read data from an Arbin .res data file and write to a sqlite file. Any data currently in the sqlite file will be erased! """ + arbin_version = mdb_get_version(input_file) + s3db = sqlite3.connect(output_file) for table in reversed(mdb_tables + mdb_5_23_tables): @@ -467,12 +507,10 @@ def convert_arbin_to_sqlite(input_file, output_file): print("Creating indices for %s..." % table) s3db.executescript(mdb_create_indices[table]) - csr = s3db.execute("SELECT Version_Schema_Field FROM Version_Table;") - version_text, = csr.fetchone() - if (version_text == "Results File 5.23"): + if arbin_version >= (5, 23): for table in mdb_5_23_tables: s3db.executescript(mdb_create_scripts[table]) - mdb_get_data(input_file, table) + mdb_get_data(s3db, input_file, table) if table in mdb_create_indices: s3db.executescript(mdb_create_indices[table]) From c25e755296e9bbacf0cd8d4b3630bea8f0fd7956 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 17 Oct 2020 17:48:03 +0300 Subject: [PATCH 3/8] Reformat lists of table names --- galvani/res2sqlite.py | 46 ++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/galvani/res2sqlite.py b/galvani/res2sqlite.py index 1899eaa..2e351f7 100755 --- a/galvani/res2sqlite.py +++ b/galvani/res2sqlite.py @@ -10,21 +10,39 @@ import argparse # The following scripts are adapted from the result of running # $ mdb-schema oracle -mdb_tables = ["Version_Table", "Global_Table", "Resume_Table", - "Channel_Normal_Table", "Channel_Statistic_Table", - "Auxiliary_Table", "Event_Table", - "Smart_Battery_Info_Table", "Smart_Battery_Data_Table"] +mdb_tables = [ + 'Version_Table', + 'Global_Table', + 'Resume_Table', + 'Channel_Normal_Table', + 'Channel_Statistic_Table', + 'Auxiliary_Table', + 'Event_Table', + 'Smart_Battery_Info_Table', + 'Smart_Battery_Data_Table', +] +mdb_5_23_tables = [ + 'MCell_Aci_Data_Table', + 'Aux_Global_Data_Table', + 'Smart_Battery_Clock_Stretch_Table', +] -mdb_tables_text = ["Version_Table", "Global_Table", "Event_Table", - "Smart_Battery_Info_Table"] -mdb_tables_numeric = ["Resume_Table", "Channel_Normal_Table", - "Channel_Statistic_Table", "Auxiliary_Table", - "Smart_Battery_Data_Table", 'MCell_Aci_Data_Table', - 'Aux_Global_Data_Table', - 'Smart_Battery_Clock_Stretch_Table'] - -mdb_5_23_tables = ['MCell_Aci_Data_Table', 'Aux_Global_Data_Table', - 'Smart_Battery_Clock_Stretch_Table'] +mdb_tables_text = { + 'Version_Table', + 'Global_Table', + 'Event_Table', + 'Smart_Battery_Info_Table', +} +mdb_tables_numeric = { + 'Resume_Table', + 'Channel_Normal_Table', + 'Channel_Statistic_Table', + 'Auxiliary_Table', + 'Smart_Battery_Data_Table', + 'MCell_Aci_Data_Table', + 'Aux_Global_Data_Table', + 'Smart_Battery_Clock_Stretch_Table', +} mdb_create_scripts = { "Version_Table": """ From c90d604096fecf0637313c12bbc1bf118bede1a3 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 17 Oct 2020 17:57:06 +0300 Subject: [PATCH 4/8] Use version number to build a list of tables to convert --- galvani/res2sqlite.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/galvani/res2sqlite.py b/galvani/res2sqlite.py index 2e351f7..5aa5551 100755 --- a/galvani/res2sqlite.py +++ b/galvani/res2sqlite.py @@ -5,6 +5,7 @@ import sqlite3 import re import csv import argparse +from copy import copy # The following scripts are adapted from the result of running @@ -515,23 +516,20 @@ def convert_arbin_to_sqlite(input_file, output_file): s3db = sqlite3.connect(output_file) - for table in reversed(mdb_tables + mdb_5_23_tables): + tables_to_convert = copy(mdb_tables) + if arbin_version >= (5, 23): + tables_to_convert.extend(mdb_5_23_tables) + + for table in reversed(tables_to_convert): s3db.execute('DROP TABLE IF EXISTS "%s";' % table) - for table in mdb_tables: + for table in tables_to_convert: s3db.executescript(mdb_create_scripts[table]) mdb_get_data(s3db, input_file, table) if table in mdb_create_indices: print("Creating indices for %s..." % table) s3db.executescript(mdb_create_indices[table]) - if arbin_version >= (5, 23): - for table in mdb_5_23_tables: - s3db.executescript(mdb_create_scripts[table]) - mdb_get_data(s3db, input_file, table) - if table in mdb_create_indices: - s3db.executescript(mdb_create_indices[table]) - print("Creating helper table for capacity and energy totals...") s3db.executescript(helper_table_script) From a1a056d3047fb9e35ae523a73072fff85ee3ae80 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 17 Oct 2020 18:10:58 +0300 Subject: [PATCH 5/8] Add two new tables that are in 5.26 but not 5.23 Can_BMS_Info_Table and Can_BMS_Data_Table --- galvani/res2sqlite.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/galvani/res2sqlite.py b/galvani/res2sqlite.py index 5aa5551..75e9e10 100755 --- a/galvani/res2sqlite.py +++ b/galvani/res2sqlite.py @@ -27,12 +27,17 @@ mdb_5_23_tables = [ 'Aux_Global_Data_Table', 'Smart_Battery_Clock_Stretch_Table', ] +mdb_5_26_tables = [ + 'Can_BMS_Info_Table', + 'Can_BMS_Data_Table', +] mdb_tables_text = { 'Version_Table', 'Global_Table', 'Event_Table', 'Smart_Battery_Info_Table', + 'Can_BMS_Info_Table', } mdb_tables_numeric = { 'Resume_Table', @@ -43,6 +48,7 @@ mdb_tables_numeric = { 'MCell_Aci_Data_Table', 'Aux_Global_Data_Table', 'Smart_Battery_Clock_Stretch_Table', + 'Can_BMS_Data_Table', } mdb_create_scripts = { @@ -257,7 +263,7 @@ CREATE TABLE Smart_Battery_Data_Table FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) ); """, - # The following tables are not present in version 1.14 + # The following tables are not present in version 1.14, but are in 5.23 'MCell_Aci_Data_Table': """ CREATE TABLE MCell_Aci_Data_Table ( @@ -326,7 +332,29 @@ CREATE TABLE Smart_Battery_Clock_Stretch_Table VCELL1 INTEGER, FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) -);"""} +);""", + # The following tables are not present in version 5.23, but are in 5.26 + 'Can_BMS_Info_Table': """ +CREATE TABLE "Can_BMS_Info_Table" + ( + Channel_Index INTEGER PRIMARY KEY, + CAN_Cfg_File_Name TEXT, + CAN_Configuration TEXT +); +""", + 'Can_BMS_Data_Table': """ +CREATE TABLE "Can_BMS_Data_Table" + ( + Test_ID INTEGER, + Data_Point INTEGER, + CAN_MV_Index INTEGER, + Signal_Value_X REAL, + PRIMARY KEY (Test_ID, Data_Point, CAN_MV_Index), + FOREIGN KEY (Test_ID, Data_Point) + REFERENCES Channel_Normal_Table (Test_ID, Data_Point) +); +""", +} mdb_create_indices = { "Channel_Normal_Table": """ @@ -519,6 +547,8 @@ def convert_arbin_to_sqlite(input_file, output_file): tables_to_convert = copy(mdb_tables) if arbin_version >= (5, 23): tables_to_convert.extend(mdb_5_23_tables) + if arbin_version >= (5, 26): + tables_to_convert.extend(mdb_5_26_tables) for table in reversed(tables_to_convert): s3db.execute('DROP TABLE IF EXISTS "%s";' % table) From 68e00a30ce39ade233315a80143ef32e1c97e2ec Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 17 Oct 2020 18:21:47 +0300 Subject: [PATCH 6/8] Add PRIMARY KEY information --- galvani/res2sqlite.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/galvani/res2sqlite.py b/galvani/res2sqlite.py index 75e9e10..4ac4739 100755 --- a/galvani/res2sqlite.py +++ b/galvani/res2sqlite.py @@ -99,7 +99,7 @@ CREATE TABLE Global_Table "Resume_Table": """ CREATE TABLE Resume_Table ( - Test_ID INTEGER REFERENCES Global_Table(Test_ID), + Test_ID INTEGER PRIMARY KEY REFERENCES Global_Table(Test_ID), Step_Index INTEGER, Cycle_Index INTEGER, Channel_Status INTEGER, @@ -149,7 +149,8 @@ CREATE TABLE Channel_Normal_Table "dV/dt" REAL, Internal_Resistance REAL, AC_Impedance REAL, - ACI_Phase_Angle REAL + ACI_Phase_Angle REAL, + PRIMARY KEY (Test_ID, Data_Point) ); """, "Channel_Statistic_Table": """ CREATE TABLE Channel_Statistic_Table @@ -160,6 +161,7 @@ CREATE TABLE Channel_Statistic_Table -- Version 1.14 ends here, version 5.23 continues Charge_Time REAL DEFAULT NULL, Discharge_Time REAL DEFAULT NULL, + PRIMARY KEY (Test_ID, Data_Point), FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) ); """, @@ -172,6 +174,7 @@ CREATE TABLE Auxiliary_Table Data_Type INTEGER, X REAL, "dX/dt" REAL, + PRIMARY KEY (Test_ID, Data_Point, Auxiliary_Index), FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) ); """, @@ -187,7 +190,7 @@ CREATE TABLE Event_Table "Smart_Battery_Info_Table": """ CREATE TABLE Smart_Battery_Info_Table ( - Test_ID INTEGER REFERENCES Global_Table(Test_ID), + Test_ID INTEGER PRIMARY KEY REFERENCES Global_Table(Test_ID), ManufacturerDate REAL, ManufacturerAccess TEXT, SpecificationInfo TEXT, @@ -257,9 +260,9 @@ CREATE TABLE Smart_Battery_Data_Table ChargingVoltage REAL DEFAULT NULL, ManufacturerData REAL DEFAULT NULL, -- Version 5.23 ends here, version 5.26 continues - BATMAN_Status INTEGER DEFAULT NULL, - DTM_PDM_Status INTEGER DEFAULT NULL, - + BATMAN_Status INTEGER DEFAULT NULL, + DTM_PDM_Status INTEGER DEFAULT NULL, + PRIMARY KEY (Test_ID, Data_Point), FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) ); """, @@ -274,6 +277,7 @@ CREATE TABLE MCell_Aci_Data_Table Phase_Shift REAL, Voltage REAL, Current REAL, + PRIMARY KEY (Test_ID, Data_Point, Cell_Index), FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) );""", @@ -284,7 +288,8 @@ CREATE TABLE Aux_Global_Data_Table Auxiliary_Index INTEGER, Data_Type INTEGER, Nickname TEXT, - Unit TEXT + Unit TEXT, + PRIMARY KEY (Channel_Index, Auxiliary_Index, Data_Type) );""", 'Smart_Battery_Clock_Stretch_Table': """ CREATE TABLE Smart_Battery_Clock_Stretch_Table @@ -330,6 +335,7 @@ CREATE TABLE Smart_Battery_Clock_Stretch_Table VCELL3 INTEGER, VCELL2 INTEGER, VCELL1 INTEGER, + PRIMARY KEY (Test_ID, Data_Point), FOREIGN KEY (Test_ID, Data_Point) REFERENCES Channel_Normal_Table (Test_ID, Data_Point) );""", From 8cfc84922bc8860a61f7a1b5dea918dea7deb3e5 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 31 Oct 2020 07:50:06 +0200 Subject: [PATCH 7/8] Add example Arbin 5.26 data file to get_testdata.sh --- get_testdata.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/get_testdata.sh b/get_testdata.sh index 792223a..9c82f97 100755 --- a/get_testdata.sh +++ b/get_testdata.sh @@ -24,4 +24,5 @@ https://files.figshare.com/1780530/121_CA_455nm_6V_30min_C01.mpt https://files.figshare.com/1780526/CV_C01.mpr https://files.figshare.com/1780527/CV_C01.mpt https://files.figshare.com/14752538/C019P-0ppb-A_C01.mpr +https://files.figshare.com/25331510/UM34_Test005E.res END_FILELIST From 23761dd5bfe554a2ef64357468f263e7e9891782 Mon Sep 17 00:00:00 2001 From: Chris Kerr Date: Sat, 31 Oct 2020 07:51:57 +0200 Subject: [PATCH 8/8] Add the new data file to test_Arbin.py --- tests/test_Arbin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_Arbin.py b/tests/test_Arbin.py index c2210eb..8d0e937 100644 --- a/tests/test_Arbin.py +++ b/tests/test_Arbin.py @@ -32,7 +32,7 @@ def test_convert_Arbin_no_mdbtools(testdata_dir, tmpdir): @pytest.mark.skipif(not have_mdbtools, reason='Reading the Arbin file requires MDBTools') -@pytest.mark.parametrize('basename', ['arbin1']) +@pytest.mark.parametrize('basename', ['arbin1', 'UM34_Test005E']) def test_convert_Arbin_to_sqlite_function(testdata_dir, tmpdir, basename): """Convert an Arbin file to SQLite using the functional interface.""" res_file = os.path.join(testdata_dir, basename + '.res')