Merge pull request #1420 from IBM/develop

chore: Trestle release

setup.cfg

@@ -32,7 +32,7 @@ install_requires =
     paramiko
     ruamel.yaml
     furl
-    pydantic[email]>=1.8.2
+    pydantic[email]>=1.8.2,<2.0.0
     python-dotenv>=0.10.4
     datamodel-code-generator[http] >= 0.11.14
     python-frontmatter

tests/trestle/core/commands/author/profile_test.py

@@ -28,6 +28,7 @@
 from tests import test_utils
 
 import trestle.common.const as const
+import trestle.core.generators as gens
 import trestle.oscal.catalog as cat
 import trestle.oscal.common as com
 import trestle.oscal.profile as prof
@@ -274,7 +275,7 @@ def test_profile_generate_assemble(
     if set_parameters_flag:
         assert set_params[2].values[0] == 'new value'
         assert set_params[1].props[0].ns == const.TRESTLE_GENERIC_NS
-        assert len(set_params) == 15
+        assert len(set_params) == 18
     else:
         # the original profile did not have ns set for this display name
         # confirm the namespace is not defined unless set_parameters_flag is True
@@ -380,7 +381,7 @@ def test_profile_ohv(required_sections: Optional[str], success: bool, ohv: bool,
         )
         set_params = profile.modify.set_parameters
 
-        assert len(set_params) == 15
+        assert len(set_params) == 18
         assert set_params[0].values[0] == 'all personnel'
         # the label is present in the header so it ends up in the set_parameter
         assert set_params[0].label == 'label from edit'
@@ -1115,3 +1116,122 @@ def test_profile_inherit(tmp_trestle_dir: pathlib.Path):
     args.output = args.profile
     prof_inherit = ProfileInherit()
     assert prof_inherit._run(args) == 2
+
+
+def test_profile_generate_assemble_parameter_aggregation(
+    tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch
+) -> None:
+    """Test the profile markdown generator."""
+    _, assembled_prof_dir, _, markdown_path = setup_profile_generate(tmp_trestle_dir, 'simple_test_profile.json')
+    yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml'
+    ac_path = markdown_path / 'ac'
+
+    nist_cat, _ = ModelUtils.load_model_for_class(tmp_trestle_dir, 'nist_cat', cat.Catalog, FileContentType.JSON)
+
+    appended_prop = {'name': 'aggregates', 'value': 'at-02_odp.01'}
+    ac_1 = nist_cat.groups[0].controls[0]
+    ac_1.params[2].props = []
+    ac_1.params[2].props.append(appended_prop)
+    appended_extra_param = {
+        'id': 'at-02_odp.01',
+        'props': [{
+            'name': 'label', 'value': 'AT-02_ODP[01]', 'class': 'sp800-53a'
+        }],
+        'label': 'frequency',
+        'guidelines': [{
+            'prose': 'blah'
+        }]
+    }
+    ac_1.params.append(appended_extra_param)
+
+    ModelUtils.save_top_level_model(nist_cat, tmp_trestle_dir, 'nist_cat', FileContentType.JSON)
+
+    # convert resolved profile catalog to markdown then assemble it after adding an item to a control
+    # generate, edit, assemble
+    test_args = f'trestle author profile-generate -n {prof_name} -o {md_name} -rs NeededExtra'.split(  # noqa E501
+    )
+    test_args.extend(['-y', str(yaml_header_path)])
+    test_args.extend(['-s', all_sections_str])
+    monkeypatch.setattr(sys, 'argv', test_args)
+
+    assert Trestle().run() == 0
+
+    fc = test_utils.FileChecker(ac_path)
+
+    assert Trestle().run() == 0
+
+    assert fc.files_unchanged()
+
+    # assemble based on set_parameters_flag
+    test_args = f'trestle author profile-assemble -n {prof_name} -m {md_name} -o {assembled_prof_name}'.split()
+    test_args.append('-sp')
+    assembled_prof_dir.mkdir()
+    monkeypatch.setattr(sys, 'argv', test_args)
+    assert Trestle().run() == 0
+
+
+def test_profile_generate_assesment_objectives(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None:
+    """Test the profile markdown generator."""
+    _, _, _, _ = setup_profile_generate(tmp_trestle_dir, 'simple_test_profile.json')
+    yaml_header_path = test_utils.YAML_TEST_DATA_PATH / 'good_simple.yaml'
+
+    profile, _ = ModelUtils.load_model_for_class(tmp_trestle_dir, 'my_prof', prof.Profile, FileContentType.JSON)
+
+    # create with-id to load at-2 control with its corresponding assesment objectives
+    with_id_at_2 = gens.generate_sample_model(prof.WithId)
+    with_id_at_2.__root__ = 'at-2'
+
+    profile.imports[0].include_controls[0].with_ids.append(with_id_at_2)
+
+    ModelUtils.save_top_level_model(profile, tmp_trestle_dir, 'my_prof', FileContentType.JSON)
+
+    nist_cat, _ = ModelUtils.load_model_for_class(tmp_trestle_dir, 'nist_cat', cat.Catalog, FileContentType.JSON)
+    # create assesment objectives json for adding it to the control in the catalog
+    assesment_objectives = {
+        'id': 'at-2_obj',
+        'name': 'assessment-objective',
+        'props': [{
+            'name': 'label', 'value': 'AT-02', 'class': 'sp800-53a'
+        }],
+        'parts': [
+            {
+                'id': 'at-2_obj.a',
+                'name': 'assessment-objective',
+                'props': [{
+                    'name': 'label', 'value': 'AT-02a.', 'class': 'sp800-53a'
+                }],
+                'parts': [
+                    {
+                        'id': 'at-2_obj.a.1-2',
+                        'name': 'assessment-objective',
+                        'props': [{
+                            'name': 'label', 'value': 'AT-02a.01[02]', 'class': 'sp800-53a'
+                        }],
+                        'prose': 'some example prose'
+                    },
+                    {
+                        'id': 'at-2_obj.a.1-3',
+                        'name': 'assessment-objective',
+                        'props': [{
+                            'name': 'label', 'value': 'AT-02a.01[03]', 'class': 'sp800-53a'
+                        }],
+                        'prose': 'some example prose'
+                    }
+                ]
+            }
+        ]
+    }
+
+    at_2 = nist_cat.groups[1].controls[1]
+    at_2.parts.append(assesment_objectives)
+    ModelUtils.save_top_level_model(nist_cat, tmp_trestle_dir, 'nist_cat', FileContentType.JSON)
+
+    # convert resolved profile catalog to markdown then assemble it after adding an item to a control
+    # generate, edit, assemble
+    test_args = f'trestle author profile-generate -n {prof_name} -o {md_name} -rs NeededExtra'.split(  # noqa E501
+    )
+    test_args.extend(['-y', str(yaml_header_path)])
+    test_args.extend(['-s', all_sections_str])
+    monkeypatch.setattr(sys, 'argv', test_args)
+
+    assert Trestle().run() == 0

tests/trestle/tasks/csv_to_oscal_cd_test.py

@@ -661,7 +661,7 @@ def test_execute_missing_param_default_value(tmp_path: pathlib.Path) -> None:
         mock_csv_reader.return_value = rows
         tgt = csv_to_oscal_cd.CsvToOscalComponentDefinition(section)
         retval = tgt.execute()
-        assert retval == TaskOutcome.FAILURE
+        assert retval == TaskOutcome.SUCCESS
 
 
 def test_execute_change_param_default_value(tmp_path: pathlib.Path) -> None:

trestle/common/const.py

@@ -250,6 +250,7 @@
 TEMPLATE_VERSION_REGEX = r'[0-9]+.[0-9]+.[0-9]+'
 
 OBJECTIVE_PART = 'objective'
+ASSESMENT_OBJECTIVE_PART = 'assessment-objective'
 TABLE_OF_PARAMS_PART = 'table_of_parameters'
 
 # extracts standalone uuid's from anywhere in string

trestle/core/catalog/catalog_reader.py

@@ -74,8 +74,8 @@ def read_additional_content(
                     # if profile_values are present, overwrite values with them
                     if const.PROFILE_VALUES in param_dict:
                         param_dict[const.VALUES] = param_dict.pop(const.PROFILE_VALUES)
-                        final_param_dict[param_id] = param_dict
-                        param_sort_map[param_id] = sort_id
+                    final_param_dict[param_id] = param_dict
+                    param_sort_map[param_id] = sort_id
         new_alters: List[prof.Alter] = []
         # fill the alters according to the control sorting order
         for key in sorted(alters_map.keys()):

trestle/core/catalog/catalog_writer.py

@@ -19,7 +19,7 @@
 
 import trestle.common.const as const
 import trestle.oscal.catalog as cat
-from trestle.common.list_utils import as_list, deep_get, none_if_empty
+from trestle.common.list_utils import as_list, deep_get, delete_list_from_list, none_if_empty
 from trestle.common.model_utils import ModelUtils
 from trestle.core.catalog.catalog_interface import CatalogInterface
 from trestle.core.catalog.catalog_merger import CatalogMerger
@@ -63,6 +63,17 @@ def write_catalog_as_profile_markdown(
 
             # get all params and vals for this control from the resolved profile catalog with block adds in effect
             control_param_dict = ControlInterface.get_control_param_dict(control, False)
+            to_delete = []
+            # removes aggregate parameters to be non-editable in markdowns
+            props_by_param_ids = {}
+            for param_id, values_dict in control_param_dict.items():
+                props_by_param_ids[param_id] = [
+                    prop for prop in as_list(values_dict.props) if prop.name == 'aggregates'
+                ]
+            to_delete = list({k: v for k, v in props_by_param_ids.items() if v != []}.keys())
+            unique_params_to_del = list(set(to_delete))
+            if unique_params_to_del:
+                delete_list_from_list(control_param_dict, unique_params_to_del)
 
             set_param_dict = self._construct_set_parameters_dict(profile_set_param_dict, control_param_dict, context)
 

trestle/core/commands/author/profile.py

@@ -144,6 +144,7 @@ def generate_markdown(
             catalog, inherited_props = ProfileResolver().get_resolved_profile_catalog_and_inherited_props(
                 trestle_root, profile_path, True, True, None, ParameterRep.LEAVE_MOUSTACHE
             )
+
             deep_set(yaml_header, [const.TRESTLE_GLOBAL_TAG, const.PROFILE, const.TITLE], profile.metadata.title)
 
             context = ControlContext.generate(ContextPurpose.PROFILE, True, trestle_root, markdown_path)
@@ -284,6 +285,33 @@ def _replace_modify_set_params(
             profile.modify.set_parameters = none_if_empty(profile.modify.set_parameters)
         return changed
 
+    @staticmethod
+    def _add_aggregated_parameter(
+        param: Any, param_dict: Dict[str, Any], control_id: str, controls: Any, param_map: Dict[str, str]
+    ) -> None:
+        """
+        Add aggregated parameter value to original parameter.
+
+        Notes:
+            None
+        """
+        # verifies aggregated param is not on grabbed param dict
+        if param.id not in list(param_dict.keys()):
+            param.values = []
+            agg_props = [prop for prop in param.props if prop.name == 'aggregates']
+            for prop in as_list(agg_props):
+                if param_dict[prop.value].get('values'):
+                    agg_param_values = param_dict[prop.value].get('values')
+                    for value in as_list(agg_param_values):
+                        param.values.append(value)
+                else:
+                    agg_param_values = [p for p in controls[control_id] if p.id == prop.value][0]
+                    param.values.append(agg_param_values.props[0].value)
+                dict_param = ModelUtils.parameter_to_dict(param, False)
+                param_dict[param.id] = dict_param
+                param_map[param.id] = control_id
+        return None
+
     @staticmethod
     def assemble_profile(
         trestle_root: pathlib.Path,
@@ -353,6 +381,13 @@ def assemble_profile(
         catalog_api = CatalogAPI(catalog=catalog, context=context)
         found_alters, param_dict, param_map = catalog_api.read_additional_content_from_md(label_as_key=True)
 
+        controls = {}
+        for group in as_list(catalog.groups):
+            for control in as_list(group.controls):
+                controls[control.id] = control.params
+        for control_id, params in controls.items():
+            for param in as_list(params):
+                ProfileAssemble._add_aggregated_parameter(param, param_dict, control_id, controls, param_map)
         # technically if allowed sections is [] it means no sections are allowed
         if allowed_sections is not None:
             for bad_part in [

trestle/core/control_writer.py

@@ -80,18 +80,20 @@ def _add_control_statement(self, control: cat.Control, group_title: str, print_g
     def _add_control_objective(self, control: cat.Control) -> None:
         if control.parts:
             for part in control.parts:
-                if part.name == 'objective':
+                if part.name == const.OBJECTIVE_PART or part.name == const.ASSESMENT_OBJECTIVE_PART:
                     self._md_file.new_paragraph()
                     heading_title = 'Control Objective'
+                    if part.name == const.ASSESMENT_OBJECTIVE_PART:
+                        heading_title = 'Control Assessment Objective'
                     self._md_file.new_header(level=2, title=heading_title)
                     self._md_file.set_indent_level(-1)
-                    self._add_part_and_its_items(control, 'objective', 'objective')
+                    self._add_part_and_its_items(control, part.name, part.name)
                     self._md_file.set_indent_level(-1)
                     return
 
     def _add_sections(self, control: cat.Control, allowed_sections: Optional[List[str]]) -> None:
         """Add the extra control sections after the main ones."""
-        skip_section_list = [const.STATEMENT, const.ITEM, const.OBJECTIVE_PART]
+        skip_section_list = [const.STATEMENT, const.ITEM, const.OBJECTIVE_PART, const.ASSESMENT_OBJECTIVE_PART]
         while True:
             _, name, title, prose = ControlInterface.get_section(control, skip_section_list)
             if not name:

trestle/tasks/csv_to_oscal_cd.py

@@ -546,18 +546,19 @@ def _create_set_parameter(self, rule_key: tuple) -> SetParameter:
         """Create create set parameters."""
         set_parameter = None
         name = self._csv_mgr.get_value(rule_key, PARAMETER_ID)
-        if name:
+        value = self._csv_mgr.get_value(rule_key, PARAMETER_VALUE_DEFAULT)
+        if name and value:
             value = self._csv_mgr.get_value(rule_key, PARAMETER_VALUE_DEFAULT)
-            if value == '':
-                row_number = self._csv_mgr.get_row_number(rule_key)
-                column_name = PARAMETER_VALUE_DEFAULT
-                text = f'row "{row_number}" missing value for "{column_name}"'
-                raise RuntimeError(text)
             values = value.split(',')
             set_parameter = SetParameter(
                 param_id=name,
                 values=values,
             )
+        elif name:
+            row_number = self._csv_mgr.get_row_number(rule_key)
+            column_name = PARAMETER_VALUE_DEFAULT
+            text = f'row "{row_number}" missing value for "{column_name}"'
+            logger.debug(text)
         return set_parameter
 
     def _get_implemented_requirement(