Merge pull request #32 from Azure/fix_markdown_conversion

Fix convert_to_markdown
Azure · Jun 11, 2020 · e7e08c9 · e7e08c9
2 parents 370c9a8 + 60cf391
commit e7e08c9
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 19 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM marvinbuss/aml-docker:1.6.0
+FROM marvinbuss/aml-docker:1.7.0
 
 LABEL maintainer="azure/gh-aml"
 

diff --git a/README.md b/README.md
@@ -149,7 +149,7 @@ The python script gets the [workspace object](https://docs.microsoft.com/en-us/p
 | run_id                       | ID of the run                                 |
 | run_url                      | URL to the run in the Azure Machine Learning Studio |
 | run_metrics                  | Metrics of the run (will only be provided if wait_for_completion is set to True) |
-| run_metrics_markdown         | Metrics of the run formatted as markdown table (will only be provided if wait_for_completion is set to True) (Diasbled for this release due to a bug) |
+| run_metrics_markdown         | Metrics of the run formatted as markdown table (will only be provided if wait_for_completion is set to True) |
 | published_pipeline_id        | Id of the published pipeline (will only be provided if you submitted a pipeline and pipeline_publish is set to True) |
 | published_pipeline_status    | Status of the published pipeline (will only be provided if you submitted a pipeline and pipeline_publish is set to True) |
 | published_pipeline_endpoint  | Endpoint of the published pipeline (will only be provided if you submitted a pipeline and pipeline_publish is set to True) |

diff --git a/code/utils.py b/code/utils.py
@@ -16,6 +16,10 @@ class AMLExperimentConfigurationException(Exception):
 
 
 def convert_to_markdown(metrics_dict):
+
+    if metrics_dict == {}:
+        return "No metrics returned"
+
     exp = list(metrics_dict.keys())
 
     experiment = exp[0]
@@ -28,17 +32,21 @@ def convert_to_markdown(metrics_dict):
     markdown += "| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A"
     for run in runs:
         # add metrics and values
-        for k, val in metrics_dict[run].items():
-            if "best_child_by_primary_metric" in k:
-                continue
-            row = f"| {run} | {k} |"
-            try:
-                val = float(val)
-                row += f" {val:.3} |"
-            except ValueError:
-                row += f" {val} |"
-            except TypeError:
-                row += f" {val} |"
+        try:
+            for k, val in metrics_dict[run].items():
+                if "best_child_by_primary_metric" in k:
+                    continue
+                row = f"| {run} | {k} |"
+                try:
+                    val = float(val)
+                    row += f" {val:.3} |"
+                except ValueError:
+                    row += f" {val} |"
+                except TypeError:
+                    row += f" {val} |"
+                markdown += row + "%0A"
+        except AttributeError:
+            row = f"| {run} | {metrics_dict[run]} |"
             markdown += row + "%0A"
     return markdown
 

diff --git a/tests/objects.py b/tests/objects.py
@@ -1,4 +1,4 @@
-markdown_conversion_input = {
+markdown_conversion_input_1 = {
     "5815b172-61fe-4e8d-baa2-e4ffe33e77d3": {
         "testkey": "testvalue"
     },
@@ -28,4 +28,72 @@
     }
 }
 
-markdown_conversion_output = """## Run Details:%0A%0A Experiment/Pipeline ID: 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 | testkey | testvalue |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_0/TimeSeries comparison_1586189163.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_1/TimeSeries comparison_1586189032.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_2/TimeSeries comparison_1586189099.png |%0A"""
+markdown_conversion_output_1 = """## Run Details:%0A%0A Experiment/Pipeline ID: 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 | testkey | testvalue |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_0/TimeSeries comparison_1586189163.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_1/TimeSeries comparison_1586189032.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_2/TimeSeries comparison_1586189099.png |%0A"""
+
+markdown_conversion_input_2 = {
+    "Training samples": 353.0,
+    "Test samples": 89.0,
+    "alpha": 0.03,
+    "mse": 3424.9003158960168
+}
+
+markdown_conversion_output_2 = """## Run Details:%0A%0A Experiment/Pipeline ID: Training samples %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| Training samples | 353.0 |%0A| Test samples | 89.0 |%0A| alpha | 0.03 |%0A| mse | 3424.9003158960168 |%0A"""
+
+markdown_conversion_input_3 = {
+    "HD_9cbf07dc-db35-4700-adc3-92253a378992": {
+        "best_child_by_primary_metric": {
+            "metric_name": [
+                "mse",
+                "mse",
+                "mse"
+            ],
+            "timestamp": [
+                "2020-03-30 07: 58: 40.110108+00: 00",
+                "2020-03-30 07: 59: 42.417631+00: 00",
+                "2020-03-30 07: 59: 42.417631+00: 00"
+            ],
+            "run_id": [
+                "HD_9cbf07dc-db35-4700-adc3-92253a378992_2",
+                "HD_9cbf07dc-db35-4700-adc3-92253a378992_1",
+                "HD_9cbf07dc-db35-4700-adc3-92253a378992_1"
+            ],
+            "metric_value": [
+                0.4896792910300023,
+                0.33284572706891424,
+                0.33284572706891424
+            ],
+            "final": [
+                False,
+                False,
+                True
+            ]
+        }
+    },
+    "HD_9cbf07dc-db35-4700-adc3-92253a378992_0": {
+        "AUC": 0.6562945178135493,
+        "RMSE": 0.05120842417634075,
+        "pAUC": 0.2812047761829226,
+        "mse": 0.9367719704449606,
+        "TimeSeries comparison": "aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_0/TimeSeries comparison_1585555145.png"
+    },
+    "HD_9cbf07dc-db35-4700-adc3-92253a378992_1": {
+        "AUC": 0.3573103979675637,
+        "pAUC": 0.5616569664904285,
+        "RMSE": 0.6161010679565609,
+        "mse": 0.33284572706891424,
+        "TimeSeries comparison": "aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_1/TimeSeries comparison_1585555146.png"
+    },
+    "HD_9cbf07dc-db35-4700-adc3-92253a378992_2": {
+        "AUC": 0.1200601984180878,
+        "pAUC": 0.5805687333099157,
+        "RMSE": 0.09691275977364577,
+        "mse": 0.4896792910300023,
+        "TimeSeries comparison": "aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_2/TimeSeries comparison_1585555076.png"
+    }
+}
+
+markdown_conversion_output_3 = """## Run Details:%0A%0A Experiment/Pipeline ID: HD_9cbf07dc-db35-4700-adc3-92253a378992 %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | AUC | 0.656 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | RMSE | 0.0512 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | pAUC | 0.281 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | mse | 0.937 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | TimeSeries comparison | aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_0/TimeSeries comparison_1585555145.png |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | AUC | 0.357 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | pAUC | 0.562 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | RMSE | 0.616 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | mse | 0.333 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | TimeSeries comparison | aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_1/TimeSeries comparison_1585555146.png |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | AUC | 0.12 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | pAUC | 0.581 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | RMSE | 0.0969 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | mse | 0.49 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | TimeSeries comparison | aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_2/TimeSeries comparison_1585555076.png |%0A"""
+
+markdown_conversion_input_4 = {}
+
+markdown_conversion_output_4 = "No metrics returned"
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -7,17 +7,36 @@
 
 from utils import AMLConfigurationException, convert_to_markdown, validate_json, load_pipeline_yaml, load_runconfig_yaml, load_runconfig_python
 from schemas import azure_credentials_schema
-from objects import markdown_conversion_input, markdown_conversion_output
+from objects import markdown_conversion_input_1, markdown_conversion_output_1, markdown_conversion_input_2, markdown_conversion_output_2, markdown_conversion_input_3, markdown_conversion_output_3, markdown_conversion_input_4, markdown_conversion_output_4
 
 
 def test_markdown_single_experiment_conversion():
     """
     Unit test to check the markdown conversion
     """
-    markdown = convert_to_markdown(
-        markdown_conversion_input
+    markdown_1 = convert_to_markdown(
+        metrics_dict=markdown_conversion_input_1
     )
-    assert markdown == markdown_conversion_output
+    print(markdown_1)
+    assert markdown_1 == markdown_conversion_output_1
+
+    markdown_2 = convert_to_markdown(
+        metrics_dict=markdown_conversion_input_2
+    )
+    print(markdown_2)
+    assert markdown_2 == markdown_conversion_output_2
+
+    markdown_3 = convert_to_markdown(
+        metrics_dict=markdown_conversion_input_3
+    )
+    print(markdown_3)
+    assert markdown_3 == markdown_conversion_output_3
+
+    markdown_4 = convert_to_markdown(
+        metrics_dict=markdown_conversion_input_4
+    )
+    print(markdown_4)
+    assert markdown_4 == markdown_conversion_output_4
 
 
 def test_validate_json_valid_inputs():