From fe85c7b27bd943300bc6ba6206eb8692741dd7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduard=20Br=C3=B6cker?= Date: Fri, 15 Nov 2019 11:24:49 +0100 Subject: [PATCH] prepare release 0.9.1 (#435) * add submodules doc * update doc to make start_bit more clear in documentation #424 * [dbc] fix dbc issues (#431) * [dbc] export for empty matrix * copy_frame handles default values now (fix #430) * [xlsx] export of coplex mutltiplexed frames no longer prohibited (fix #403) --- docs/api.rst | 29 ++++++++++++++++++++++- src/canmatrix/canmatrix.py | 5 +++- src/canmatrix/copy.py | 13 ++++++++--- src/canmatrix/formats/dbc.py | 40 +++++++++++++++++--------------- src/canmatrix/formats/xlsx.py | 3 +-- src/canmatrix/tests/test_copy.py | 28 ++++++++++++++++++++++ src/canmatrix/tests/test_dbc.py | 15 +++++++----- 7 files changed, 101 insertions(+), 32 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index f20d315d..21fb88d0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -6,7 +6,34 @@ ____________ .. automodule:: canmatrix.canmatrix :members: - + +.. automodule:: canmatrix.Ecu + :members: + +.. automodule:: canmatrix.Frame + :members: + +.. automodule:: canmatrix.Signal + :members: + +.. automodule:: canmatrix.SignalGroup + :members: + +.. automodule:: canmatrix.DecodedSignal + :members: + +.. automodule:: canmatrix.unpack_bitstring + :members: + +.. automodule:: canmatrix.pack_bitstring + :members: + +.. automodule:: canmatrix.ArbitrationId + :members: + +.. automodule:: canmatrix.Define + :members: + cancluster.py _____________ diff --git a/src/canmatrix/canmatrix.py b/src/canmatrix/canmatrix.py index 5c08b5a0..ed643b96 100644 --- a/src/canmatrix/canmatrix.py +++ b/src/canmatrix/canmatrix.py @@ -128,7 +128,8 @@ class Signal(object): Signal has following attributes: * name - * start_bit, size (in Bits) + * start_bit (internal start_bit, see get/set_startbit also) + * size (in Bits) * is_little_endian (1: Intel, 0: Motorola) * is_signed (bool) * factor, offset, min, max @@ -987,6 +988,8 @@ def add_attribute(self, attribute, value): self.attributes[attribute] = str(value) except UnicodeDecodeError: self.attributes[attribute] = value + if type(self.attributes[attribute]) == str: + self.attributes[attribute] = self.attributes[attribute].strip() def del_attribute(self, attribute): # type: (str) -> typing.Any diff --git a/src/canmatrix/copy.py b/src/canmatrix/copy.py index 5c728b93..c551981a 100644 --- a/src/canmatrix/copy.py +++ b/src/canmatrix/copy.py @@ -170,13 +170,16 @@ def copy_frame(frame_id, source_db, target_db): copy_ecu(source_ecu, source_db, target_db) # copy all frame-defines - attributes = frame.attributes - for attribute in attributes: + for attribute in source_db.frame_defines: if attribute not in target_db.frame_defines: target_db.add_frame_defines( copy.deepcopy(attribute), copy.deepcopy(source_db.frame_defines[attribute].definition)) target_db.add_define_default( copy.deepcopy(attribute), copy.deepcopy(source_db.frame_defines[attribute].defaultValue)) + # only default value exists in source but is different to default value in target + if attribute not in frame.attributes and \ + frame.attribute(attribute, source_db) != frame.attribute(attribute, target_db): + target_db.frame_by_id(frame.arbitration_id).add_attribute(attribute, frame.attribute(attribute, source_db)) # update enum data types if needed: if source_db.frame_defines[attribute].type == 'ENUM': temp_attr = frame.attribute(attribute, db=source_db) @@ -187,7 +190,7 @@ def copy_frame(frame_id, source_db, target_db): # trigger all signals of Frame for sig in frame.signals: # delete all 'unknown' attributes - for attribute in sig.attributes: + for attribute in source_db.signal_defines: target_db.add_signal_defines( copy.deepcopy(attribute), copy.deepcopy(source_db.signal_defines[attribute].definition)) target_db.add_define_default( @@ -198,5 +201,9 @@ def copy_frame(frame_id, source_db, target_db): if temp_attr not in target_db.signal_defines[attribute].values: target_db.signal_defines[attribute].values.append(copy.deepcopy(temp_attr)) target_db.signal_defines[attribute].update() + # only default value exists in source but is different to default value in target + if attribute not in sig.attributes and \ + sig.attribute(attribute, source_db) != sig.attribute(attribute, target_db): + target_db.frame_by_id(frame.arbitration_id).signal_by_name(sig.name).add_attribute(attribute, sig.attribute(attribute, source_db)) return True diff --git a/src/canmatrix/formats/dbc.py b/src/canmatrix/formats/dbc.py index 78c0a0d8..08bc953e 100644 --- a/src/canmatrix/formats/dbc.py +++ b/src/canmatrix/formats/dbc.py @@ -247,13 +247,15 @@ def dump(in_db, f, **options): name += str(duplicate_signal_counter[name] - 1) output_names[frame][signal] = name - if max([x.cycle_time for x in db.frames]) > 0: - db.add_frame_defines("GenMsgCycleTime", 'INT 0 65535') - if max([x.cycle_time for y in db.frames for x in y.signals]) > 0: - db.add_signal_defines("GenSigCycleTime", 'INT 0 65535') + if len(db.frames) > 0: + if max([x.cycle_time for x in db.frames]) > 0: + db.add_frame_defines("GenMsgCycleTime", 'INT 0 65535') + if len([s for fr in db.frames for s in fr.signals]) > 0: + if max([x.cycle_time for y in db.frames for x in y.signals]) > 0: + db.add_signal_defines("GenSigCycleTime", 'INT 0 65535') - if max([x.initial_value for y in db.frames for x in y.signals]) > 0 or min([x.initial_value for y in db.frames for x in y.signals]) < 0: - db.add_signal_defines("GenSigStartValue", 'FLOAT 0 100000000000') + if max([x.initial_value for y in db.frames for x in y.signals]) > 0 or min([x.initial_value for y in db.frames for x in y.signals]) < 0: + db.add_signal_defines("GenSigStartValue", 'FLOAT 0 100000000000') @@ -507,7 +509,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None comment += "\n" + l.decode(dbc_comment_encoding).replace('\\"', '"') except: logger.error("Error decoding line: %d (%s)" % (i, line)) - if re.match('.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None: + if re.match(r'.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None: follow_up = _FollowUps.NOTHING if signal is not None: signal.add_comment(comment[:-1].strip()[:-1]) @@ -517,7 +519,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None comment += "\n" + l.decode(dbc_comment_encoding).replace('\\"', '"') except: logger.error("Error decoding line: %d (%s)" % (i, line)) - if re.match('.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None: + if re.match(r'.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None: follow_up = _FollowUps.NOTHING if frame is not None: frame.add_comment(comment[:-1].strip()[:-1]) @@ -528,7 +530,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None l.decode(dbc_comment_encoding).replace('\\"', '"') except: logger.error("Error decoding line: %d (%s)" % (i, line)) - if re.match('.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None: + if re.match(r'.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None: follow_up = _FollowUps.NOTHING if board_unit is not None: board_unit.add_comment(comment[:-1].strip()[:-1]) @@ -785,7 +787,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None substring = decoded[7:].strip() define_type = substring[:3] substring = substring[3:].strip() - pattern = r"^\"(.+?)\" +(.+);" + pattern = r"^\"(.+?)\" +(.+); *" regexp = re.compile(pattern) regexp_raw = re.compile(pattern.encode(dbc_import_encoding)) temp = regexp.match(substring) @@ -816,23 +818,23 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None tempba = regexp.match(decoded) if tempba.group(1).strip().startswith("BO_ "): - regexp = re.compile(r"^BA_ +\"(.+?)\" +BO_ +(\d+) +(.+) *;") + regexp = re.compile(r"^BA_ +\"(.+?)\" +BO_ +(\d+) +(.+) *; *") temp = regexp.match(decoded) get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(2)))).add_attribute( temp.group(1), temp.group(3)) elif tempba.group(1).strip().startswith("SG_ "): - regexp = re.compile(r"^BA_ +\"(.+?)\" +SG_ +(\d+) +(\w+) +(.+) *;") + regexp = re.compile(r"^BA_ +\"(.+?)\" +SG_ +(\d+) +(\w+) +(.+) *; *") temp = regexp.match(decoded) if temp is not None: get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(2)))).signal_by_name( temp.group(3)).add_attribute(temp.group(1), temp.group(4)) elif tempba.group(1).strip().startswith("EV_ "): - regexp = re.compile(r"^BA_ +\"(.+?)\" +EV_ +(\w+) +(.*) *;") + regexp = re.compile(r"^BA_ +\"(.+?)\" +EV_ +(\w+) +(.*) *; *") temp = regexp.match(decoded) if temp is not None: db.add_env_attribute(temp.group(2), temp.group(1), temp.group(3)) elif tempba.group(1).strip().startswith("BU_ "): - regexp = re.compile(r"^BA_ +\"(.*?)\" +BU_ +(\w+) +(.+) *;") + regexp = re.compile(r"^BA_ +\"(.*?)\" +BU_ +(\w+) +(.+) *; *") temp = regexp.match(decoded) db.ecu_by_name( temp.group(2)).add_attribute( @@ -840,13 +842,13 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None temp.group(3)) else: regexp = re.compile( - r"^BA_ +\"([A-Za-z0-9\-_]+)\" +([\"\w\-\.]+) *;") + r"^BA_ +\"([A-Za-z0-9\-_]+)\" +([\"\w\-\.]+) *; *") temp = regexp.match(decoded) if temp: db.add_attribute(temp.group(1), temp.group(2)) elif decoded.startswith("SIG_GROUP_ "): - regexp = re.compile(r"^SIG_GROUP_ +(\w+) +(\w+) +(\w+) +\:(.*) *;") + regexp = re.compile(r"^SIG_GROUP_ +(\w+) +(\w+) +(\w+) +\:(.*) *; *") temp = regexp.match(decoded) frame = get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(1)))) if frame is not None: @@ -854,7 +856,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None frame.add_signal_group(temp.group(2), temp.group(3), signal_array) # todo wrong annotation in canmatrix? Id is a string? elif decoded.startswith("SIG_VALTYPE_ "): - regexp = re.compile(r"^SIG_VALTYPE_ +(\w+) +(\w+)\s*\:(.*) *;") + regexp = re.compile(r"^SIG_VALTYPE_ +(\w+) +(\w+)\s*\:(.*) *; *") temp = regexp.match(decoded) frame = get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(1)))) if frame: @@ -872,7 +874,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None db.add_define_default(temp.group(1), temp_raw.group(2).decode(dbc_import_encoding)) elif decoded.startswith("SG_MUL_VAL_ "): - pattern = r"^SG_MUL_VAL_ +([0-9]+) +([\w\-]+) +([\w\-]+) +(.*) *;" + pattern = r"^SG_MUL_VAL_ +([0-9]+) +([\w\-]+) +([\w\-]+) +(.*) *; *" regexp = re.compile(pattern) temp = regexp.match(decoded) if temp: @@ -891,7 +893,7 @@ def add_frame_by_id(new_frame): # type: (canmatrix.Frame) -> None mux_val_max_number = int(mux_val_max) signal.mux_val_grp.append([mux_val_min_number, mux_val_max_number]) elif decoded.startswith("EV_ "): - pattern = r"^EV_ +([\w\-\_]+?) *\: +([0-9]+) +\[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] +\"(.*?)\" +([0-9.+\-eE]+) +([0-9.+\-eE]+) +([\w\-]+?) +(.*);" + pattern = r"^EV_ +([\w\-\_]+?) *\: +([0-9]+) +\[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] +\"(.*?)\" +([0-9.+\-eE]+) +([0-9.+\-eE]+) +([\w\-]+?) +(.*); *" regexp = re.compile(pattern) temp = regexp.match(decoded) diff --git a/src/canmatrix/formats/xlsx.py b/src/canmatrix/formats/xlsx.py index 4803655c..39463aec 100644 --- a/src/canmatrix/formats/xlsx.py +++ b/src/canmatrix/formats/xlsx.py @@ -199,8 +199,7 @@ def dump(db, filename, **options): logger.debug("DEBUG: Length of db.frames is %d", len(db.frames)) for frame in db.frames: if frame.is_complex_multiplexed: - logger.error("Export complex multiplexers is not supported - ignoring frame %s", frame.name) - continue + logger.error("Export complex multiplexers is not supported - frame %s might be uncomplete", frame.name) frame_hash[int(frame.arbitration_id.id)] = frame # set row to first Frame (row = 0 is header) diff --git a/src/canmatrix/tests/test_copy.py b/src/canmatrix/tests/test_copy.py index a7a94f29..9ba74a4b 100644 --- a/src/canmatrix/tests/test_copy.py +++ b/src/canmatrix/tests/test_copy.py @@ -75,3 +75,31 @@ def test_copy_ecu_with_attributes(): assert len(matrix1.ecus) == 1 assert matrix1.ecu_by_name("ECU") is not None assert matrix1.ecu_by_name("ECU").attribute("Node Address") == 42 + +def test_copy_frame_default_attributes(): + source = canmatrix.canmatrix.CanMatrix() + frame1 = canmatrix.canmatrix.Frame("Frame1", arbitration_id=1) + signal = canmatrix.canmatrix.Signal("Signal1") + frame1.add_signal(canmatrix.canmatrix.Signal("SomeSignal")) + frame1.add_signal(signal) + source.add_frame(frame1) + source.add_frame_defines("some_attribute", "STRING") + source.add_define_default("some_attribute", "source_frame_default") + source.add_signal_defines("some_signal_attribute", "STRING") + source.add_define_default("some_signal_attribute", "source_sig_default") + + #test if default value only defined in source and copied to target + target = canmatrix.canmatrix.CanMatrix() + canmatrix.copy.copy_frame(frame1.arbitration_id, source, target) + assert target.frames[0].attribute("some_attribute", target) == "source_frame_default" + assert target.frames[0].signals[0].attribute("some_signal_attribute", target) == "source_sig_default" + + # test if define already exists, but has another default value: + target2 = canmatrix.canmatrix.CanMatrix() + target2.add_frame_defines("some_attribute", "STRING") + target2.add_define_default("some_attribute", "target_frame_default") + target2.add_signal_defines("some_signal_attribute", "STRING") + target2.add_define_default("some_signal_attribute", "target_sig_default") + canmatrix.copy.copy_frame(frame1.arbitration_id, source, target2) + assert target2.frames[0].attribute("some_attribute", target2) == "source_frame_default" + assert target2.frames[0].signals[0].attribute("some_signal_attribute", target2) == "source_sig_default" diff --git a/src/canmatrix/tests/test_dbc.py b/src/canmatrix/tests/test_dbc.py index beb3e728..2b005a4a 100644 --- a/src/canmatrix/tests/test_dbc.py +++ b/src/canmatrix/tests/test_dbc.py @@ -333,15 +333,18 @@ def test_j1939_frametype(): matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8") assert matrix.frames[0].is_j1939 == False -def test_signal_definition_with_spaces_iss358(): - dbc = io.BytesIO(textwrap.dedent(u'''\ - BU_: someOtherEcu - BO_ 123 someFrame: 1 someOtherEcu - SG_ AccSts : 62|3@0+ (1.0, 0.0) [0.0|0.0] "" VDDM +def test_attributes_with_spaces_before_semicolumn(): + dbc = io.BytesIO(textwrap.dedent(u'''\ + BO_ 8 Frame_1: 8 Vector__XXX + BO_ 9 Frame_2: 8 Vector__XXX + BA_DEF_ BO_ "someAttribute" STRING ; + BA_ "someAttribute" BO_ 8 "str" ; + BA_DEF_DEF_ "someAttribute" "asd" ; ''').encode('utf-8')) matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8") - + assert matrix.frames[0].attributes["someAttribute"] == 'str' + assert matrix.frames[1].attribute("someAttribute", matrix) == 'asd' def test_cycle_time_handling(): dbc = io.BytesIO(textwrap.dedent(u'''\