Merge branch 'develop' into feat/add-authoring-public-api

tests/data/author/governed_folders/instance_with_diff_heading_levels/architecture.md

@@ -0,0 +1,59 @@
+---
+authors:
+  - Tim
+  - Jane
+  - Sally
+owner: Joe
+valid:
+  from: 2020-01-01
+  to: 2099-12-31
+---
+
+# Vulnerability Management (VULN) Defect Checks
+## 0. Vulnerability Management Workflow
+### 0.1 Data Sources
+### 0.2 Fetchers
+### 0.3 Data Store
+### 0.4 Policy Engine
+### 0.5 Ticketing System
+## 1. Facts Data Model
+### 1.1 Devices
+#### Server
+#### KubernetesCluster
+#### ContainerImage
+### 1.2 Vulnerabilities
+#### ResourceScan
+#### ResourceScanFinding
+#### ResourceScanResult
+### 1.3 Thresholds
+#### CISOOverride
+#### CISAKEV
+### 1.4 Risks
+#### VulnDeviations
+### 1.5 Scanner Definition
+#### ScannerConfiguration
+## 2. Defect Checks
+### Sub-capability: Reduce Software/ Firmware Vulnerabilities
+#### Vulnerable Software/ Firmware
+##### Purpose
+##### Assessment Criteria
+###### Inputs
+###### Rules
+####### vuln_prod_os_scan_duedate_check
+######## Type
+######## Rationale Statement
+######## Impact Statement
+######## Implementation Description
+######## Audit Procedure(s)
+######## Remediation Procedure(s)
+######## Parameters
+####### vuln_prod_os_scan_warning_duedate_check_warning
+######## Type
+######## Rationale Statement
+######## Impact Statement
+######## Implementation Description
+######## Audit Procedure(s)
+######## Remediation Procedure(s)
+######## Parameters
+###### Additional Outputs
+##### Assessment Objectives

tests/data/author/governed_folders/template_folder_headling_levels/architecture.md

@@ -0,0 +1,30 @@
+---
+authors:
+  - Tim
+  - Jane
+  - Sally
+owner: Joe
+valid:
+  from: 2020-01-01
+  to: 2099-12-31
+---
+
+# { Security Capability Name } Defect Checks
+## 1. Facts Data Model
+### Sub-Capability: { _insert name of subcapability_}
+## 2. Defect Checks
+### Sub-capability: { _insert sub-capability name_}
+#### { _insert defect check name_}
+##### Assessment Criteria
+###### Inputs
+###### Rules
+####### { Rule Name}
+######## Type
+######## Rationale Statement
+######## Impact Statement
+######## Implementation Description
+######## Audit Procedure(s)
+######## Remediation Procedure(s)
+######## Parameters
+###### Additional Outputs
+##### Assessment Objectives

tests/data/csv/bp.sample.v3.csv

@@ -0,0 +1,12 @@
+"Reference_Id","Rule_Id","Rule_Description","Check_Id","Check_Description","Fetcher","Fetcher_Description","Profile_Source","Profile_Description","Component_Type","Control_Id_List","Component_Title","Component_Description","Parameter_Id","Parameter_Description","Parameter_Value_Default","Parameter_Value_Alternatives","Namespace"
+"column description","column description","column description",,,,,,,,,,,,,,,
+3000020,"account_owner_authorized_ip_range_configured","Ensure authorized IP ranges are configured by the account owner","account_owner_authorized_ip_range_configured","Check whether authorized IP ranges are configured by the account owner",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","sc-7_smt.a sc-7_smt.b sc-7.3 sc-7.4_smt.a sc-7.5 ia-3","IAM","IAM",,,,,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000021,"iam_admin_role_users_per_account_maxcount","Ensure there are no more than # IAM administrators configured per account","iam_admin_role_users_per_account_maxcount","Check whether there are no more than # IAM administrators configured per account",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-6 ac-5_smt.c","IAM","IAM","allowed_admins_per_account","Maximum allowed administrators per",10,10,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000022,"iam_cos_public_access_disabled","Ensure Cloud Object Storage public access is disabled in IAM settings (not applicable to ACLs managed using S3 APIs)","iam_cos_public_access_disabled","Check whether Cloud Object Storage public access is disabled in IAM settings (not applicable to ACLs managed using S3 APIs)",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-3 ac-4 ac-6 sc-7_smt.a sc-7_smt.b sc-7.4_smt.a ac-14_smt.a cm-7_smt.a cm-7_smt.b","IAM","IAM",,,,,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000023,"iam_account_owner_no_api_key","Ensure the account owner does not have an IBM Cloud API key created in IAM","iam_account_owner_no_api_key","Check whether the account owner does not have an IBM Cloud API key created in IAM",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-2_smt.d ac-3 ac-5_smt.c ac-6","IAM","IAM",,,,,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000024,"iam_api_keys_rotation_configured","Ensure IBM Cloud API keys that are managed in IAM are rotated at least every # days","iam_api_keys_rotation_configured","Check whether IBM Cloud API keys that are managed in IAM are rotated at least every # days",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ia-5_smt.g","IAM","IAM","api_keys_rotated_days","API Keys Rotated","x, y, z",,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000027,"iam_account_owner_api_key_restrictions_configured","Ensure permissions for API key creation are limited and configured in IAM settings for the account owner","iam_account_owner_api_key_restrictions_configured","Check whether permissions for API key creation are limited and configured in IAM settings for the account owner",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-2_smt.d ac-3 ac-5_smt.c ac-6","IAM","IAM",,,,,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000029,"iam_admin_role__user_maxcount","Ensure IAM-enabled services have no more than # users with the IAM administrator role","iam_admin_role__user_maxcount","Check whether IAM-enabled services have no more than # users with the IAM administrator role",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-6 ac-5_smt.c ia-7","IAM","IAM","no_of_admins_for_iam","Maximum no of IAM user","a, b, c",,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000030,"iam_serviceID_policies_attached_to_access_groups_or_roles","Ensure IAM policies for service IDs are attached only to groups or roles","iam_serviceID_policies_attached_to_access_groups_or_roles","Check whether IAM policies for service IDs are attached only to groups or roles",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-3 ac-6 ac-2_smt.d ac-5_smt.c ia-7","IAM","IAM",,,,,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000031,"iam_logDNA_enabled","Ensure Identity and Access Management (IAM) is enabled with audit logging","iam_logDNA_enabled","Check whether Identity and Access Management (IAM) is enabled with audit logging",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","au-2_smt.a au-2_smt.d si-4_smt.a si-4_smt.b si-4_smt.c au-12_smt.a au-12_smt.b au-12_smt.c au-3 au-8_smt.a au-8_smt.b au-8.1_smt.a au-8.1_smt.b ca-7_smt.d","IAM","IAM",,,,,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"
+3000032,"iam_admin_role_serviceid_maxcount","Ensure IAM-enabled services have no more than # service IDs with the IAM administrator role","iam_admin_role_serviceid_maxcount","Check whether IAM-enabled services have no more than # service IDs with the IAM administrator role",,,"https://github.com/usnistgov/oscal-content/blob/main/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_HIGH-baseline_profile.json","NIST Special Publication 800-53 Revision 5 HIGH IMPACT BASELINE","Service","ac-6 ac-5_smt.c ia-7","IAM","IAM","no_of_service_id_admins_for_iam","Maximum no of IAM Service ID","3, 4, 5",,"http://ibm.github.io/compliance-trestle/schemas/oscal/cd"

tests/trestle/core/commands/author/folders_test.py

@@ -674,3 +674,27 @@ def test_drawio_versioning_validation(
     monkeypatch.setattr(sys, 'argv', command_string_validate_content.split())
     rc = trestle.cli.Trestle().run()
     assert rc == 0
+
+
+def test_heading_levels_hierarchy(
+    testdata_dir: pathlib.Path, tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch
+) -> None:
+    """Test behaviour when validating drawio instance."""
+    task_template_folder = tmp_trestle_dir / '.trestle/author/test_task/'
+    test_template_folder = testdata_dir / 'author/governed_folders/template_folder_headling_levels'
+    test_instances_folder = testdata_dir / 'author/governed_folders/instance_with_diff_heading_levels'
+    task_instance_folder = tmp_trestle_dir / 'test_task/folder_1'
+
+    hidden_file = testdata_dir / pathlib.Path(
+        'author/governed_folders/template_folder_with_drawio/.hidden_does_not_affect'
+    )
+    test_utils.make_file_hidden(hidden_file)
+
+    test_utils.copy_tree_or_file_with_hidden(test_template_folder, task_template_folder)
+
+    shutil.copytree(test_instances_folder, task_instance_folder)
+
+    command_string_validate_content = 'trestle author folders validate -tn test_task -hv'
+    monkeypatch.setattr(sys, 'argv', command_string_validate_content.split())
+    rc = trestle.cli.Trestle().run()
+    assert rc == 0

tests/trestle/core/markdown/markdown_validator_test.py

@@ -55,13 +55,6 @@
             False,
             False
         ),
-        (
-            pathlib.Path('tests/data/author/0.0.1/test_1_md_format/template.md'),
-            pathlib.Path('tests/data/author/0.0.1/test_1_md_format/bad_instance_reordered.md'),
-            False,
-            False,
-            False
-        ),
         (
             pathlib.Path('tests/data/author/0.0.1/test_1_md_format/template.md'),
             pathlib.Path('tests/data/author/0.0.1/test_1_md_format/bad_instance_missing_heading.md'),
@@ -125,13 +118,6 @@
             False,
             False
         ),
-        (
-            pathlib.Path('tests/data/author/0.0.1/test_4_md_format_extras/template.md'),
-            pathlib.Path('tests/data/author/0.0.1/test_4_md_format_extras/bad_instance_reordered.md'),
-            False,
-            False,
-            False
-        ),
         (
             pathlib.Path('tests/data/author/0.0.1/test_4_md_format_extras/template.md'),
             pathlib.Path('tests/data/author/0.0.1/test_4_md_format_extras/bad_instance_missing_heading.md'),

tests/trestle/tasks/csv_to_oscal_cd_test.py

@@ -442,6 +442,34 @@ def test_execute_bp_sample(tmp_path: pathlib.Path) -> None:
     _validate_bp(tmp_path)
 
 
+def test_execute_bp3_sample(tmp_path: pathlib.Path) -> None:
+    """Test execute bp3 sample."""
+    _, section = _get_config_section_init(tmp_path, 'test-csv-to-oscal-cd-bp.config')
+    section['csv-file'] = 'tests/data/csv/bp.sample.v3.csv'
+    tgt = csv_to_oscal_cd.CsvToOscalComponentDefinition(section)
+    retval = tgt.execute()
+    assert retval == TaskOutcome.SUCCESS
+    # read component-definition
+    fp = pathlib.Path(tmp_path) / 'component-definition.json'
+    cd = ComponentDefinition.oscal_read(fp)
+    # spot check
+    component = cd.components[0]
+    assert len(component.props) == 59
+    assert len(component.control_implementations) == 1
+    ci = component.control_implementations[0]
+    assert len(ci.set_parameters) == 4
+    assert len(ci.set_parameters[0].values) == 1
+    assert len(ci.set_parameters[1].values) == 3
+    assert ci.set_parameters[1].values[0] == 'x'
+    assert ci.set_parameters[1].values[1] == 'y'
+    assert ci.set_parameters[1].values[2] == 'z'
+    assert len(ci.set_parameters[2].values) == 3
+    assert len(ci.set_parameters[3].values) == 3
+    assert ci.set_parameters[3].values[0] == '3'
+    assert ci.set_parameters[3].values[1] == '4'
+    assert ci.set_parameters[3].values[2] == '5'
+
+
 def test_execute_bp_cd(tmp_path: pathlib.Path) -> None:
     """Test execute bp cd."""
     _, section = _get_config_section_init(tmp_path, 'test-csv-to-oscal-cd-bp.config')

trestle/core/markdown/markdown_validator.py

@@ -21,6 +21,7 @@
 
 import trestle.core.markdown.markdown_const as md_const
 from trestle.common.err import TrestleError
+from trestle.common.list_utils import as_list
 from trestle.core.commands.author.consts import START_TEMPLATE_VERSION, TEMPLATE_VERSION_HEADER
 from trestle.core.markdown.docs_markdown_node import DocsMarkdownNode
 
@@ -202,30 +203,25 @@ def _validate_headings(self, instance: pathlib.Path, template_keys: List[str], i
             )
             return False
         template_header_pointer = 0
+        present_keys = []
         for key in instance_keys:
             if template_header_pointer >= len(template_keys):
                 break
-            if key in template_keys and key != template_keys[template_header_pointer]:
-                logger.warning(
-                    f'Headings in the instance: {instance} were shuffled or modified. '
-                    f'\nInstance does not have required template heading '
-                    f'\"{template_keys[template_header_pointer]}\". '
-                    f'Check if this heading was modified/present in the instance.'
-                    f'\nPlease note that no changes to template headings are allowed, '
-                    f'including extra spaces.'
-                )
-                return False
-            elif key in template_keys and key == template_keys[template_header_pointer]:
+            if key in template_keys and key not in present_keys:
+                present_keys.append(template_keys[template_keys.index(key)])
                 template_header_pointer += 1
             elif re.search(md_const.SUBSTITUTION_REGEX, template_keys[template_header_pointer]) is not None:
+                present_keys.append(template_keys[template_header_pointer])
                 template_header_pointer += 1  # skip headers with substitutions
-        if template_header_pointer != len(template_keys):
+        diff_keys = set(template_keys) - set(present_keys)
+        if template_header_pointer != len(template_keys) and len(diff_keys) > 0:
             logger.info(
                 f'Headings in the instance: {instance} were removed. '
                 f'Expected {len(template_keys)} headings, but found only {template_header_pointer}.'
             )
+            for result in as_list(diff_keys):
+                logger.info(f'Heading {result} in the instance: {instance} was removed or not present ')
             return False
-
         return True
 
     @classmethod

trestle/tasks/csv_to_oscal_cd.py

@@ -576,14 +576,25 @@ def _get_control_implementation(
         component.control_implementations.append(control_implementation)
         return control_implementation
 
+    def _str_to_list(self, value: str) -> List[str]:
+        """Transform string to list."""
+        rval = []
+        if ',' in value:
+            values = value.split(',')
+            # remove leading/trailing whitespace
+            for v in values:
+                rval.append(v.strip())
+        else:
+            rval.append(value)
+        return rval
+
     def _create_set_parameter(self, rule_key: tuple) -> SetParameter:
         """Create create set parameters."""
         set_parameter = None
         name = self._csv_mgr.get_value(rule_key, PARAMETER_ID)
         value = self._csv_mgr.get_value(rule_key, PARAMETER_VALUE_DEFAULT)
         if name and value:
-            value = self._csv_mgr.get_value(rule_key, PARAMETER_VALUE_DEFAULT)
-            values = value.split(',')
+            values = self._str_to_list(value)
             set_parameter = SetParameter(
                 param_id=name,
                 values=values,