prepare release 0.9.1 (#435)

* add submodules doc
* update doc to make start_bit more clear in documentation #424
* [dbc] fix dbc issues (#431)
* [dbc] export for empty matrix
* copy_frame handles default values now (fix #430)
* [xlsx] export of coplex mutltiplexed frames no longer prohibited (fix #403)

docs/api.rst

@@ -6,7 +6,34 @@ ____________
 
 .. automodule:: canmatrix.canmatrix
     :members:
-
+
+.. automodule:: canmatrix.Ecu
+    :members:
+
+.. automodule:: canmatrix.Frame
+    :members:
+
+.. automodule:: canmatrix.Signal
+    :members:
+
+.. automodule:: canmatrix.SignalGroup
+    :members:
+
+.. automodule:: canmatrix.DecodedSignal
+    :members:
+
+.. automodule:: canmatrix.unpack_bitstring
+    :members:
+
+.. automodule:: canmatrix.pack_bitstring
+    :members:
+
+.. automodule:: canmatrix.ArbitrationId
+    :members:
+
+.. automodule:: canmatrix.Define
+    :members:
+
 cancluster.py
 _____________
 

src/canmatrix/canmatrix.py

@@ -128,7 +128,8 @@ class Signal(object):
     Signal has following attributes:
 
     * name
-    * start_bit, size (in Bits)
+    * start_bit (internal start_bit, see get/set_startbit also)
+    * size (in Bits)
     * is_little_endian (1: Intel, 0: Motorola)
     * is_signed (bool)
     * factor, offset, min, max
@@ -987,6 +988,8 @@ def add_attribute(self, attribute, value):
             self.attributes[attribute] = str(value)
         except UnicodeDecodeError:
             self.attributes[attribute] = value
+        if type(self.attributes[attribute]) == str:
+            self.attributes[attribute] = self.attributes[attribute].strip()
 
     def del_attribute(self, attribute):
         # type: (str) -> typing.Any

src/canmatrix/copy.py

@@ -170,13 +170,16 @@ def copy_frame(frame_id, source_db, target_db):
                     copy_ecu(source_ecu, source_db, target_db)
 
         # copy all frame-defines
-        attributes = frame.attributes
-        for attribute in attributes:
+        for attribute in source_db.frame_defines:
             if attribute not in target_db.frame_defines:
                 target_db.add_frame_defines(
                     copy.deepcopy(attribute), copy.deepcopy(source_db.frame_defines[attribute].definition))
                 target_db.add_define_default(
                     copy.deepcopy(attribute), copy.deepcopy(source_db.frame_defines[attribute].defaultValue))
+            # only default value exists in source but is different to default value in target
+            if attribute not in frame.attributes and \
+                    frame.attribute(attribute, source_db) != frame.attribute(attribute, target_db):
+                target_db.frame_by_id(frame.arbitration_id).add_attribute(attribute, frame.attribute(attribute, source_db))
             # update enum data types if needed:
             if source_db.frame_defines[attribute].type == 'ENUM':
                 temp_attr = frame.attribute(attribute, db=source_db)
@@ -187,7 +190,7 @@ def copy_frame(frame_id, source_db, target_db):
         # trigger all signals of Frame
         for sig in frame.signals:
             # delete all 'unknown' attributes
-            for attribute in sig.attributes:
+            for attribute in source_db.signal_defines:
                 target_db.add_signal_defines(
                     copy.deepcopy(attribute), copy.deepcopy(source_db.signal_defines[attribute].definition))
                 target_db.add_define_default(
@@ -198,5 +201,9 @@ def copy_frame(frame_id, source_db, target_db):
                     if temp_attr not in target_db.signal_defines[attribute].values:
                         target_db.signal_defines[attribute].values.append(copy.deepcopy(temp_attr))
                         target_db.signal_defines[attribute].update()
+            # only default value exists in source but is different to default value in target
+            if attribute not in sig.attributes and \
+                    sig.attribute(attribute, source_db) != sig.attribute(attribute, target_db):
+                target_db.frame_by_id(frame.arbitration_id).signal_by_name(sig.name).add_attribute(attribute, sig.attribute(attribute, source_db))
 
     return True

src/canmatrix/formats/dbc.py

@@ -247,13 +247,15 @@ def dump(in_db, f, **options):
                 name += str(duplicate_signal_counter[name] - 1)
             output_names[frame][signal] = name
 
-    if max([x.cycle_time for x in db.frames]) > 0:
-        db.add_frame_defines("GenMsgCycleTime", 'INT 0 65535')
-    if max([x.cycle_time for y in db.frames for x in y.signals]) > 0:
-        db.add_signal_defines("GenSigCycleTime", 'INT 0 65535')
+    if len(db.frames) > 0:
+        if max([x.cycle_time for x in db.frames]) > 0:
+            db.add_frame_defines("GenMsgCycleTime", 'INT 0 65535')
+        if len([s for fr in db.frames for s in fr.signals]) > 0:
+            if max([x.cycle_time for y in db.frames for x in y.signals]) > 0:
+                db.add_signal_defines("GenSigCycleTime", 'INT 0 65535')
 
-    if max([x.initial_value for y in db.frames for x in y.signals]) > 0 or min([x.initial_value for y in db.frames for x in y.signals]) < 0:
-        db.add_signal_defines("GenSigStartValue", 'FLOAT 0 100000000000')
+            if max([x.initial_value for y in db.frames for x in y.signals]) > 0 or min([x.initial_value for y in db.frames for x in y.signals]) < 0:
+                db.add_signal_defines("GenSigStartValue", 'FLOAT 0 100000000000')
 
 
 
@@ -507,7 +509,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                     comment += "\n" + l.decode(dbc_comment_encoding).replace('\\"', '"')
                 except:
                     logger.error("Error decoding line: %d (%s)" % (i, line))
-                if re.match('.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None:
+                if re.match(r'.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None:
                     follow_up = _FollowUps.NOTHING
                     if signal is not None:
                         signal.add_comment(comment[:-1].strip()[:-1])
@@ -517,7 +519,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                     comment += "\n" + l.decode(dbc_comment_encoding).replace('\\"', '"')
                 except:
                     logger.error("Error decoding line: %d (%s)" % (i, line))
-                if re.match('.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None:
+                if re.match(r'.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None:
                     follow_up = _FollowUps.NOTHING
                     if frame is not None:
                         frame.add_comment(comment[:-1].strip()[:-1])
@@ -528,7 +530,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                         l.decode(dbc_comment_encoding).replace('\\"', '"')
                 except:
                     logger.error("Error decoding line: %d (%s)" % (i, line))
-                if re.match('.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None:
+                if re.match(r'.*" *;\Z',l.decode(dbc_import_encoding).strip()) is not None:
                     follow_up = _FollowUps.NOTHING
                     if board_unit is not None:
                         board_unit.add_comment(comment[:-1].strip()[:-1])
@@ -785,7 +787,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                 substring = decoded[7:].strip()
                 define_type = substring[:3]
                 substring = substring[3:].strip()
-                pattern = r"^\"(.+?)\" +(.+);"
+                pattern = r"^\"(.+?)\" +(.+); *"
                 regexp = re.compile(pattern)
                 regexp_raw = re.compile(pattern.encode(dbc_import_encoding))
                 temp = regexp.match(substring)
@@ -816,45 +818,45 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                 tempba = regexp.match(decoded)
 
                 if tempba.group(1).strip().startswith("BO_ "):
-                    regexp = re.compile(r"^BA_ +\"(.+?)\" +BO_ +(\d+) +(.+) *;")
+                    regexp = re.compile(r"^BA_ +\"(.+?)\" +BO_ +(\d+) +(.+) *; *")
                     temp = regexp.match(decoded)
                     get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(2)))).add_attribute(
                         temp.group(1), temp.group(3))
                 elif tempba.group(1).strip().startswith("SG_ "):
-                    regexp = re.compile(r"^BA_ +\"(.+?)\" +SG_ +(\d+) +(\w+) +(.+) *;")
+                    regexp = re.compile(r"^BA_ +\"(.+?)\" +SG_ +(\d+) +(\w+) +(.+) *; *")
                     temp = regexp.match(decoded)
                     if temp is not None:
                         get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(2)))).signal_by_name(
                             temp.group(3)).add_attribute(temp.group(1), temp.group(4))
                 elif tempba.group(1).strip().startswith("EV_ "):
-                    regexp = re.compile(r"^BA_ +\"(.+?)\" +EV_ +(\w+) +(.*) *;")
+                    regexp = re.compile(r"^BA_ +\"(.+?)\" +EV_ +(\w+) +(.*) *; *")
                     temp = regexp.match(decoded)
                     if temp is not None:
                         db.add_env_attribute(temp.group(2), temp.group(1), temp.group(3))
                 elif tempba.group(1).strip().startswith("BU_ "):
-                    regexp = re.compile(r"^BA_ +\"(.*?)\" +BU_ +(\w+) +(.+) *;")
+                    regexp = re.compile(r"^BA_ +\"(.*?)\" +BU_ +(\w+) +(.+) *; *")
                     temp = regexp.match(decoded)
                     db.ecu_by_name(
                         temp.group(2)).add_attribute(
                         temp.group(1),
                         temp.group(3))
                 else:
                     regexp = re.compile(
-                        r"^BA_ +\"([A-Za-z0-9\-_]+)\" +([\"\w\-\.]+) *;")
+                        r"^BA_ +\"([A-Za-z0-9\-_]+)\" +([\"\w\-\.]+) *; *")
                     temp = regexp.match(decoded)
                     if temp:
                         db.add_attribute(temp.group(1), temp.group(2))
 
             elif decoded.startswith("SIG_GROUP_ "):
-                regexp = re.compile(r"^SIG_GROUP_ +(\w+) +(\w+) +(\w+) +\:(.*) *;")
+                regexp = re.compile(r"^SIG_GROUP_ +(\w+) +(\w+) +(\w+) +\:(.*) *; *")
                 temp = regexp.match(decoded)
                 frame = get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(1))))
                 if frame is not None:
                     signal_array = temp.group(4).split(' ')
                     frame.add_signal_group(temp.group(2), temp.group(3), signal_array)  # todo wrong annotation in canmatrix? Id is a string?
 
             elif decoded.startswith("SIG_VALTYPE_ "):
-                regexp = re.compile(r"^SIG_VALTYPE_ +(\w+) +(\w+)\s*\:(.*) *;")
+                regexp = re.compile(r"^SIG_VALTYPE_ +(\w+) +(\w+)\s*\:(.*) *; *")
                 temp = regexp.match(decoded)
                 frame = get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(int(temp.group(1))))
                 if frame:
@@ -872,7 +874,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                     db.add_define_default(temp.group(1),
                                           temp_raw.group(2).decode(dbc_import_encoding))
             elif decoded.startswith("SG_MUL_VAL_ "):
-                pattern = r"^SG_MUL_VAL_ +([0-9]+) +([\w\-]+) +([\w\-]+) +(.*) *;"
+                pattern = r"^SG_MUL_VAL_ +([0-9]+) +([\w\-]+) +([\w\-]+) +(.*) *; *"
                 regexp = re.compile(pattern)
                 temp = regexp.match(decoded)
                 if temp:
@@ -891,7 +893,7 @@ def add_frame_by_id(new_frame):  # type: (canmatrix.Frame) -> None
                             mux_val_max_number = int(mux_val_max)
                             signal.mux_val_grp.append([mux_val_min_number, mux_val_max_number])
             elif decoded.startswith("EV_ "):
-                pattern = r"^EV_ +([\w\-\_]+?) *\: +([0-9]+) +\[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] +\"(.*?)\" +([0-9.+\-eE]+) +([0-9.+\-eE]+) +([\w\-]+?) +(.*);"
+                pattern = r"^EV_ +([\w\-\_]+?) *\: +([0-9]+) +\[([0-9.+\-eE]+)\|([0-9.+\-eE]+)\] +\"(.*?)\" +([0-9.+\-eE]+) +([0-9.+\-eE]+) +([\w\-]+?) +(.*); *"
                 regexp = re.compile(pattern)
                 temp = regexp.match(decoded)
 

src/canmatrix/formats/xlsx.py

@@ -199,8 +199,7 @@ def dump(db, filename, **options):
     logger.debug("DEBUG: Length of db.frames is %d", len(db.frames))
     for frame in db.frames:
         if frame.is_complex_multiplexed:
-            logger.error("Export complex multiplexers is not supported - ignoring frame %s", frame.name)
-            continue
+            logger.error("Export complex multiplexers is not supported - frame %s might be uncomplete", frame.name)
         frame_hash[int(frame.arbitration_id.id)] = frame
 
     # set row to first Frame (row = 0 is header)

src/canmatrix/tests/test_copy.py

@@ -75,3 +75,31 @@ def test_copy_ecu_with_attributes():
     assert len(matrix1.ecus) == 1
     assert matrix1.ecu_by_name("ECU") is not None
     assert matrix1.ecu_by_name("ECU").attribute("Node Address") == 42
+
+def test_copy_frame_default_attributes():
+    source = canmatrix.canmatrix.CanMatrix()
+    frame1 = canmatrix.canmatrix.Frame("Frame1", arbitration_id=1)
+    signal = canmatrix.canmatrix.Signal("Signal1")
+    frame1.add_signal(canmatrix.canmatrix.Signal("SomeSignal"))
+    frame1.add_signal(signal)
+    source.add_frame(frame1)
+    source.add_frame_defines("some_attribute", "STRING")
+    source.add_define_default("some_attribute", "source_frame_default")
+    source.add_signal_defines("some_signal_attribute", "STRING")
+    source.add_define_default("some_signal_attribute", "source_sig_default")
+
+    #test if default value only defined in source and copied to target
+    target = canmatrix.canmatrix.CanMatrix()
+    canmatrix.copy.copy_frame(frame1.arbitration_id, source, target)
+    assert target.frames[0].attribute("some_attribute", target) == "source_frame_default"
+    assert target.frames[0].signals[0].attribute("some_signal_attribute", target) == "source_sig_default"
+
+    # test if define already exists, but has another default value:
+    target2 = canmatrix.canmatrix.CanMatrix()
+    target2.add_frame_defines("some_attribute", "STRING")
+    target2.add_define_default("some_attribute", "target_frame_default")
+    target2.add_signal_defines("some_signal_attribute", "STRING")
+    target2.add_define_default("some_signal_attribute", "target_sig_default")
+    canmatrix.copy.copy_frame(frame1.arbitration_id, source, target2)
+    assert target2.frames[0].attribute("some_attribute", target2) == "source_frame_default"
+    assert target2.frames[0].signals[0].attribute("some_signal_attribute", target2) == "source_sig_default"

src/canmatrix/tests/test_dbc.py

@@ -333,15 +333,18 @@ def test_j1939_frametype():
     matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8")
     assert matrix.frames[0].is_j1939 == False
 
-def test_signal_definition_with_spaces_iss358():
-    dbc = io.BytesIO(textwrap.dedent(u'''\
-    BU_: someOtherEcu
 
-    BO_ 123 someFrame: 1 someOtherEcu
-    SG_ AccSts : 62|3@0+ (1.0, 0.0) [0.0|0.0] "" VDDM
+def test_attributes_with_spaces_before_semicolumn():
+    dbc = io.BytesIO(textwrap.dedent(u'''\
+    BO_ 8 Frame_1: 8 Vector__XXX
+    BO_ 9 Frame_2: 8 Vector__XXX
+    BA_DEF_ BO_ "someAttribute" STRING ;
+    BA_ "someAttribute" BO_ 8 "str" ;
+    BA_DEF_DEF_ "someAttribute" "asd" ;
     ''').encode('utf-8'))
     matrix = canmatrix.formats.dbc.load(dbc, dbcImportEncoding="utf8")
-
+    assert matrix.frames[0].attributes["someAttribute"] == 'str'
+    assert matrix.frames[1].attribute("someAttribute", matrix) == 'asd'
 
 def test_cycle_time_handling():
     dbc = io.BytesIO(textwrap.dedent(u'''\