chris/galvani
1
0
Fork 0
mirror of https://github.com/echemdata/galvani.git synced 2025-12-14 01:15:34 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
3c904db04ea673a5c2a2f26b7e0a68c5b9339ac5
galvani/galvani/res2sqlite.py

638 lines
21 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/python
# SPDX-FileCopyrightText: 2013-2020 Christopher Kerr <chris.kerr@mykolab.ch>
#
# SPDX-License-Identifier: GPL-3.0-or-later
import subprocess as sp
import sqlite3
import re
import csv
import argparse
from copy import copy
# The following scripts are adapted from the result of running
# $ mdb-schema <result.res> oracle
mdb_tables = [
"Version_Table",
"Global_Table",
"Resume_Table",
"Channel_Normal_Table",
"Channel_Statistic_Table",
"Auxiliary_Table",
"Event_Table",
"Smart_Battery_Info_Table",
"Smart_Battery_Data_Table",
]
mdb_5_23_tables = [
"MCell_Aci_Data_Table",
"Aux_Global_Data_Table",
"Smart_Battery_Clock_Stretch_Table",
]
mdb_5_26_tables = [
"Can_BMS_Info_Table",
"Can_BMS_Data_Table",
]
mdb_tables_text = {
"Version_Table",
"Global_Table",
"Event_Table",
"Smart_Battery_Info_Table",
"Can_BMS_Info_Table",
}
mdb_tables_numeric = {
"Resume_Table",
"Channel_Normal_Table",
"Channel_Statistic_Table",
"Auxiliary_Table",
"Smart_Battery_Data_Table",
"MCell_Aci_Data_Table",
"Aux_Global_Data_Table",
"Smart_Battery_Clock_Stretch_Table",
"Can_BMS_Data_Table",
}
mdb_create_scripts = {
"Version_Table": """
CREATE TABLE Version_Table
(
Version_Schema_Field TEXT,
Version_Comments_Field TEXT
); """,
"Global_Table": """
CREATE TABLE Global_Table
(
Test_ID INTEGER PRIMARY KEY,
Test_Name TEXT,
Channel_Index INTEGER,
Start_DateTime REAL,
DAQ_Index INTEGER,
Channel_Type INTEGER,
Creator TEXT,
Comments TEXT,
Schedule_File_Name TEXT,
Channel_Number INTEGER,
Mapped_Aux_Voltage_Number INTEGER,
Mapped_Aux_Temperature_Number INTEGER,
Mapped_Aux_Pressure_Number INTEGER,
Mapped_Aux_PH_Number INTEGER,
Mapped_Aux_Flow_Rate_CNumber INTEGER,
Applications_Path TEXT,
Log_ChanStat_Data_Flag INTEGER,
Log_Aux_Data_Flag INTEGER,
Log_Event_Data_Flag INTEGER,
Log_Smart_Battery_Data_Flag INTEGER,
-- The following items are in 5.26 but not in 5.23
Log_Can_BMS_Data_Flag INTEGER DEFAULT NULL,
Software_Version TEXT DEFAULT NULL,
Serial_Number TEXT DEFAULT NULL,
Schedule_Version TEXT DEFAULT NULL,
MASS REAL DEFAULT NULL,
Specific_Capacity REAL DEFAULT NULL,
Capacity REAL DEFAULT NULL,
-- Item_ID exists in all versions
Item_ID TEXT,
-- These items are in 5.26 and 5.23 but not in 1.14
Mapped_Aux_Conc_CNumber INTEGER DEFAULT NULL,
Mapped_Aux_DI_CNumber INTEGER DEFAULT NULL,
Mapped_Aux_DO_CNumber INTEGER DEFAULT NULL
); """,
"Resume_Table": """
CREATE TABLE Resume_Table
(
Test_ID INTEGER PRIMARY KEY REFERENCES Global_Table(Test_ID),
Step_Index INTEGER,
Cycle_Index INTEGER,
Channel_Status INTEGER,
Test_Time REAL,
Step_Time REAL,
CCapacity REAL,
DCapacity REAL,
CEnergy REAL,
DEnergy REAL,
TC_Time1 REAL,
TC_Time2 REAL,
TC_Time3 REAL,
TC_Time4 REAL,
TC_CCapacity1 REAL,
TC_CCapacity2 REAL,
TC_DCapacity1 REAL,
TC_DCapacity2 REAL,
TC_CEnergy1 REAL,
TC_CEnergy2 REAL,
TC_DEnergy1 REAL,
TC_DEnergy2 REAL,
MV_Counter1 REAL,
MV_Counter2 REAL,
MV_Counter3 REAL,
MV_Counter4 REAL,
-- Version 1.14 ends here, version 5.23 continues
Charge_Time REAL DEFAULT NULL,
Discharge_Time REAL DEFAULT NULL
); """,
"Channel_Normal_Table": """
CREATE TABLE Channel_Normal_Table
(
Test_ID INTEGER REFERENCES Global_Table(Test_ID),
Data_Point INTEGER,
Test_Time REAL,
Step_Time REAL,
DateTime REAL,
Step_Index INTEGER,
Cycle_Index INTEGER,
Is_FC_Data INTEGER,
Current REAL,
Voltage REAL,
Charge_Capacity REAL,
Discharge_Capacity REAL,
Charge_Energy REAL,
Discharge_Energy REAL,
"dV/dt" REAL,
Internal_Resistance REAL,
AC_Impedance REAL,
ACI_Phase_Angle REAL,
PRIMARY KEY (Test_ID, Data_Point)
); """,
"Channel_Statistic_Table": """
CREATE TABLE Channel_Statistic_Table
(
Test_ID INTEGER,
Data_Point INTEGER,
Vmax_On_Cycle REAL,
-- Version 1.14 ends here, version 5.23 continues
Charge_Time REAL DEFAULT NULL,
Discharge_Time REAL DEFAULT NULL,
PRIMARY KEY (Test_ID, Data_Point),
FOREIGN KEY (Test_ID, Data_Point)
REFERENCES Channel_Normal_Table (Test_ID, Data_Point)
); """,
"Auxiliary_Table": """
CREATE TABLE Auxiliary_Table
(
Test_ID INTEGER,
Data_Point INTEGER,
Auxiliary_Index INTEGER,
Data_Type INTEGER,
X REAL,
"dX/dt" REAL,
PRIMARY KEY (Test_ID, Data_Point, Auxiliary_Index),
FOREIGN KEY (Test_ID, Data_Point)
REFERENCES Channel_Normal_Table (Test_ID, Data_Point)
); """,
"Event_Table": """
CREATE TABLE Event_Table
(
Test_ID INTEGER REFERENCES Global_Table(Test_ID),
DateTime REAL,
Test_Time REAL,
Event_Type INTEGER,
Event_Describe TEXT
); """,
"Smart_Battery_Info_Table": """
CREATE TABLE Smart_Battery_Info_Table
(
Test_ID INTEGER PRIMARY KEY REFERENCES Global_Table(Test_ID),
ManufacturerDate REAL,
ManufacturerAccess TEXT,
SpecificationInfo TEXT,
FullChargeCapacity REAL,
ChargingCurrent REAL,
ChargingVoltage REAL,
DesignCapacity REAL,
DesignVoltage REAL,
SerialNumber INTEGER,
ManufacturerName TEXT,
DeviceName TEXT,
DeviceChemistry TEXT,
ManufacturerData TEXT,
-- Version 1.14 ends here, version 5.23 continues
Frequency INTEGER DEFAULT NULL
); """,
"Smart_Battery_Data_Table": """
CREATE TABLE Smart_Battery_Data_Table
(
Test_ID INTEGER,
Data_Point INTEGER,
RemainingCapacityAlarm REAL,
RemainingTimeAlarm INTEGER,
BatteryMode INTEGER,
AtRate REAL,
AtRateTimeToFull INTEGER,
AtRateTimeToEmpty INTEGER,
AtRateOK INTEGER,
Temperature REAL,
Voltage REAL,
Current REAL,
AverageCurrent REAL,
MaxError INTEGER,
RelativeStateOfCharge INTEGER,
AbsoluteStateOfCharge INTEGER,
RemainingCapacity REAL,
RunTimeToEmpty INTEGER,
AverageTimeToEmpty INTEGER,
AverageTimeToFull INTEGER,
BatteryStatus INTEGER,
CycleCount INTEGER,
-- Version 1.14 ends here, version 5.23 continues
Pack_Status INTEGER DEFAULT NULL,
Pack_Configuration INTEGER DEFAULT NULL,
VCELL4 REAL DEFAULT NULL,
VCELL3 REAL DEFAULT NULL,
VCELL2 REAL DEFAULT NULL,
VCELL1 REAL DEFAULT NULL,
ManufacturerAccess TEXT DEFAULT NULL,
FullChargeCapacity REAL DEFAULT NULL,
BroadCast INTEGER DEFAULT NULL,
GPIO1 INTEGER DEFAULT NULL,
GPIO2 INTEGER DEFAULT NULL,
OptVCELL4 REAL DEFAULT NULL,
OptVCELL3 REAL DEFAULT NULL,
OptVCELL2 REAL DEFAULT NULL,
OptVCELL1 REAL DEFAULT NULL,
OMF1 INTEGER DEFAULT NULL,
OMF2 INTEGER DEFAULT NULL,
OMF3 INTEGER DEFAULT NULL,
OMF4 INTEGER DEFAULT NULL,
OMF5 INTEGER DEFAULT NULL,
FTEMP INTEGER DEFAULT NULL,
STATUS INTEGER DEFAULT NULL,
FET_TEMP INTEGER DEFAULT NULL,
ChargingCurrent REAL DEFAULT NULL,
ChargingVoltage REAL DEFAULT NULL,
ManufacturerData REAL DEFAULT NULL,
-- Version 5.23 ends here, version 5.26 continues
BATMAN_Status INTEGER DEFAULT NULL,
DTM_PDM_Status INTEGER DEFAULT NULL,
PRIMARY KEY (Test_ID, Data_Point),
FOREIGN KEY (Test_ID, Data_Point)
REFERENCES Channel_Normal_Table (Test_ID, Data_Point)
); """,
# The following tables are not present in version 1.14, but are in 5.23
"MCell_Aci_Data_Table": """
CREATE TABLE MCell_Aci_Data_Table
(
Test_ID INTEGER,
Data_Point INTEGER,
Cell_Index INTEGER,
ACI REAL,
Phase_Shift REAL,
Voltage REAL,
Current REAL,
PRIMARY KEY (Test_ID, Data_Point, Cell_Index),
FOREIGN KEY (Test_ID, Data_Point)
REFERENCES Channel_Normal_Table (Test_ID, Data_Point)
);""",
"Aux_Global_Data_Table": """
CREATE TABLE Aux_Global_Data_Table
(
Channel_Index INTEGER,
Auxiliary_Index INTEGER,
Data_Type INTEGER,
Nickname TEXT,
Unit TEXT,
PRIMARY KEY (Channel_Index, Auxiliary_Index, Data_Type)
);""",
"Smart_Battery_Clock_Stretch_Table": """
CREATE TABLE Smart_Battery_Clock_Stretch_Table
(
Test_ID INTEGER,
Data_Point INTEGER,
ManufacturerAccess INTEGER,
RemainingCapacityAlarm INTEGER,
RemainingTimeAlarm INTEGER,
BatteryMode INTEGER,
AtRate INTEGER,
AtRateTimeToFull INTEGER,
AtRateTimeToEmpty INTEGER,
AtRateOK INTEGER,
Temperature INTEGER,
Voltage INTEGER,
Current INTEGER,
AverageCurrent INTEGER,
MaxError INTEGER,
RelativeStateOfCharge INTEGER,
AbsoluteStateOfCharge INTEGER,
RemainingCapacity INTEGER,
FullChargeCapacity INTEGER,
RunTimeToEmpty INTEGER,
AverageTimeToEmpty INTEGER,
AverageTimeToFull INTEGER,
ChargingCurrent INTEGER,
ChargingVoltage INTEGER,
BatteryStatus INTEGER,
CycleCount INTEGER,
DesignCapacity INTEGER,
DesignVoltage INTEGER,
SpecificationInfo INTEGER,
ManufacturerDate INTEGER,
SerialNumber INTEGER,
ManufacturerName INTEGER,
DeviceName INTEGER,
DeviceChemistry INTEGER,
ManufacturerData INTEGER,
Pack_Status INTEGER,
Pack_Configuration INTEGER,
VCELL4 INTEGER,
VCELL3 INTEGER,
VCELL2 INTEGER,
VCELL1 INTEGER,
PRIMARY KEY (Test_ID, Data_Point),
FOREIGN KEY (Test_ID, Data_Point)
REFERENCES Channel_Normal_Table (Test_ID, Data_Point)
);""",
# The following tables are not present in version 5.23, but are in 5.26
"Can_BMS_Info_Table": """
CREATE TABLE "Can_BMS_Info_Table"
(
Channel_Index INTEGER PRIMARY KEY,
CAN_Cfg_File_Name TEXT,
CAN_Configuration TEXT
);
""",
"Can_BMS_Data_Table": """
CREATE TABLE "Can_BMS_Data_Table"
(
Test_ID INTEGER,
Data_Point INTEGER,
CAN_MV_Index INTEGER,
Signal_Value_X REAL,
PRIMARY KEY (Test_ID, Data_Point, CAN_MV_Index),
FOREIGN KEY (Test_ID, Data_Point)
REFERENCES Channel_Normal_Table (Test_ID, Data_Point)
);
""",
}
mdb_create_indices = {
"Channel_Normal_Table": """
CREATE UNIQUE INDEX data_point_index ON Channel_Normal_Table (Test_ID, Data_Point);
CREATE INDEX voltage_index ON Channel_Normal_Table (Test_ID, Voltage);
CREATE INDEX test_time_index ON Channel_Normal_Table (Test_ID, Test_Time);
"""
}
helper_table_script = """
CREATE TEMPORARY TABLE capacity_helper(
Test_ID INTEGER NOT NULL,
Cycle_Index INTEGER NOT NULL,
Charge_Capacity REAL NOT NULL,
Discharge_Capacity REAL NOT NULL,
Charge_Energy REAL NOT NULL,
Discharge_Energy REAL NOT NULL,
FOREIGN KEY (Test_ID, Cycle_Index)
REFERENCES Channel_Normal_Table (Test_ID, Cycle_Index)
);
INSERT INTO capacity_helper
SELECT Test_ID, Cycle_Index,
max(Charge_Capacity), max(Discharge_Capacity),
max(Charge_Energy), max(Discharge_Energy)
FROM Channel_Normal_Table
GROUP BY Test_ID, Cycle_Index;
-- ## Alternative way of selecting ##
-- select *
-- from Channel_Normal_Table as a join Channel_Normal_Table as b
-- on (a.Test_ID = b.Test_ID and a.Data_Point = b.Data_Point + 1
-- and a.Charge_Capacity < b.Charge_Capacity);
DROP TABLE IF EXISTS Capacity_Sum_Table;
CREATE TABLE Capacity_Sum_Table(
Test_ID INTEGER NOT NULL,
Cycle_Index INTEGER NOT NULL,
Charge_Capacity_Sum REAL NOT NULL,
Discharge_Capacity_Sum REAL NOT NULL,
Charge_Energy_Sum REAL NOT NULL,
Discharge_Energy_Sum REAL NOT NULL,
FOREIGN KEY (Test_ID, Cycle_Index)
REFERENCES Channel_Normal_Table (Test_ID, Cycle_Index)
);
INSERT INTO Capacity_Sum_Table
SELECT a.Test_ID, a.Cycle_Index,
total(b.Charge_Capacity), total(b.Discharge_Capacity),
total(b.Charge_Energy), total(b.Discharge_Energy)
FROM capacity_helper AS a LEFT JOIN capacity_helper AS b
ON (a.Test_ID = b.Test_ID AND a.Cycle_Index > b.Cycle_Index)
GROUP BY a.Test_ID, a.Cycle_Index;
DROP TABLE capacity_helper;
CREATE VIEW IF NOT EXISTS Capacity_View
AS SELECT Test_ID, Data_Point, Test_Time, Step_Time, DateTime,
Step_Index, Cycle_Index, Current, Voltage, "dV/dt",
( (Discharge_Capacity + Discharge_Capacity_Sum)
- (Charge_Capacity + Charge_Capacity_Sum) ) AS Net_Capacity,
( (Discharge_Capacity + Discharge_Capacity_Sum)
+ (Charge_Capacity + Charge_Capacity_Sum) ) AS Gross_Capacity,
( (Discharge_Energy + Discharge_Energy_Sum)
- (Charge_Energy + Charge_Energy_Sum) ) AS Net_Energy,
( (Discharge_Energy + Discharge_Energy_Sum)
+ (Charge_Energy + Charge_Energy_Sum) ) AS Gross_Energy
FROM Channel_Normal_Table NATURAL JOIN Capacity_Sum_Table;
"""
def mdb_get_data_text(s3db, filename, table):
print("Reading %s..." % table)
insert_pattern = re.compile(
r"""INSERT INTO "\w+" \([^)]+?\) VALUES (\((('[^']*')|"[^"]*"|[^')])+?\),?\s*)+;\n""",
re.IGNORECASE,
)
try:
# Initialize values to avoid NameError in except clause
mdb_output = ""
insert_match = None
with sp.Popen(
["mdb-export", "-I", "postgres", filename, table],
bufsize=-1,
stdin=sp.DEVNULL,
stdout=sp.PIPE,
universal_newlines=True,
) as mdb_sql:
mdb_output = mdb_sql.stdout.read()
while len(mdb_output) > 0:
insert_match = insert_pattern.match(mdb_output)
s3db.execute(insert_match.group())
mdb_output = mdb_output[insert_match.end():]
mdb_output += mdb_sql.stdout.read()
s3db.commit()
except OSError as e:
if e.errno == 2:
raise RuntimeError(
"Could not locate the `mdb-export` executable. "
"Check that mdbtools is properly installed."
)
else:
raise
except BaseException:
print("Error while importing %s" % table)
if mdb_output:
print("Remaining mdb-export output:", mdb_output)
if insert_match:
print("insert_re match:", insert_match)
raise
def mdb_get_data_numeric(s3db, filename, table):
print("Reading %s..." % table)
try:
with sp.Popen(
["mdb-export", filename, table],
bufsize=-1,
stdin=sp.DEVNULL,
stdout=sp.PIPE,
universal_newlines=True,
) as mdb_sql:
mdb_csv = csv.reader(mdb_sql.stdout)
mdb_headers = next(mdb_csv)
quoted_headers = ['"%s"' % h for h in mdb_headers]
joined_headers = ", ".join(quoted_headers)
joined_placemarks = ", ".join(["?" for h in mdb_headers])
insert_stmt = 'INSERT INTO "{0}" ({1}) VALUES ({2});'.format(
table,
joined_headers,
joined_placemarks,
)
s3db.executemany(insert_stmt, mdb_csv)
s3db.commit()
except OSError as e:
if e.errno == 2:
raise RuntimeError(
"Could not locate the `mdb-export` executable. "
"Check that mdbtools is properly installed."
)
else:
raise
def mdb_get_data(s3db, filename, table):
if table in mdb_tables_text:
mdb_get_data_text(s3db, filename, table)
elif table in mdb_tables_numeric:
mdb_get_data_numeric(s3db, filename, table)
else:
raise ValueError(
"'%s' is in neither mdb_tables_text nor mdb_tables_numeric" % table
)
def mdb_get_version(filename):
"""Get the version number from an Arbin .res file.
Reads the Version_Table and parses the version from Version_Schema_Field.
"""
print("Reading version number...")
try:
with sp.Popen(
["mdb-export", filename, "Version_Table"],
bufsize=-1,
stdin=sp.DEVNULL,
stdout=sp.PIPE,
universal_newlines=True,
) as mdb_sql:
mdb_csv = csv.reader(mdb_sql.stdout)
mdb_headers = next(mdb_csv)
mdb_values = next(mdb_csv)
try:
next(mdb_csv)
except StopIteration:
pass
else:
raise ValueError(
"Version_Table of %s lists multiple versions" % filename
)
except OSError as e:
if e.errno == 2:
raise RuntimeError(
"Could not locate the `mdb-export` executable. "
"Check that mdbtools is properly installed."
)
else:
raise
if "Version_Schema_Field" not in mdb_headers:
raise ValueError(
"Version_Table of %s does not contain a Version_Schema_Field column"
% filename
)
version_fields = dict(zip(mdb_headers, mdb_values))
version_text = version_fields["Version_Schema_Field"]
version_match = re.fullmatch("Results File ([.0-9]+)", version_text)
if not version_match:
raise ValueError(
'File version "%s" did not match expected format' % version_text
)
version_string = version_match.group(1)
version_tuple = tuple(map(int, version_string.split(".")))
return version_tuple
def convert_arbin_to_sqlite(input_file, output_file=None):
"""Read data from an Arbin .res data file and write to a sqlite file.
Any data currently in an sqlite file at `output_file` will be erased!
Parameters:
input_file (str): The path to the Arbin .res file to read from.
output_file (str or None): The path to the sqlite file to write to; if None,
return a `sqlite3.Connection` into an in-memory database.
Returns:
None or sqlite3.Connection
"""
arbin_version = mdb_get_version(input_file)
if output_file is None:
output_file = ":memory:"
s3db = sqlite3.connect(output_file)
tables_to_convert = copy(mdb_tables)
if arbin_version >= (5, 23):
tables_to_convert.extend(mdb_5_23_tables)
if arbin_version >= (5, 26):
tables_to_convert.extend(mdb_5_26_tables)
for table in reversed(tables_to_convert):
s3db.execute('DROP TABLE IF EXISTS "%s";' % table)
for table in tables_to_convert:
s3db.executescript(mdb_create_scripts[table])
mdb_get_data(s3db, input_file, table)
if table in mdb_create_indices:
print("Creating indices for %s..." % table)
s3db.executescript(mdb_create_indices[table])
print("Creating helper table for capacity and energy totals...")
s3db.executescript(helper_table_script)
print("Vacuuming database...")
s3db.executescript("VACUUM; ANALYZE;")
if output_file == ":memory:":
return s3db
s3db.close()
def main(argv=None):
parser = argparse.ArgumentParser(
description="Convert Arbin .res files to sqlite3 databases using mdb-export",
)
parser.add_argument("input_file", type=str) # need file name to pass to sp.Popen
parser.add_argument(
"output_file", type=str
) # need file name to pass to sqlite3.connect
args = parser.parse_args(argv)
convert_arbin_to_sqlite(args.input_file, args.output_file)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink