diff --git a/Dockerfile b/Dockerfile index 9655597..542942f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM marvinbuss/aml-docker:1.6.0 +FROM marvinbuss/aml-docker:1.7.0 LABEL maintainer="azure/gh-aml" diff --git a/README.md b/README.md index f42e623..e80a60b 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ The python script gets the [workspace object](https://docs.microsoft.com/en-us/p | run_id | ID of the run | | run_url | URL to the run in the Azure Machine Learning Studio | | run_metrics | Metrics of the run (will only be provided if wait_for_completion is set to True) | -| run_metrics_markdown | Metrics of the run formatted as markdown table (will only be provided if wait_for_completion is set to True) (Diasbled for this release due to a bug) | +| run_metrics_markdown | Metrics of the run formatted as markdown table (will only be provided if wait_for_completion is set to True) | | published_pipeline_id | Id of the published pipeline (will only be provided if you submitted a pipeline and pipeline_publish is set to True) | | published_pipeline_status | Status of the published pipeline (will only be provided if you submitted a pipeline and pipeline_publish is set to True) | | published_pipeline_endpoint | Endpoint of the published pipeline (will only be provided if you submitted a pipeline and pipeline_publish is set to True) | diff --git a/code/utils.py b/code/utils.py index 2ac8fe5..b4a3bbe 100644 --- a/code/utils.py +++ b/code/utils.py @@ -16,6 +16,10 @@ class AMLExperimentConfigurationException(Exception): def convert_to_markdown(metrics_dict): + + if metrics_dict == {}: + return "No metrics returned" + exp = list(metrics_dict.keys()) experiment = exp[0] @@ -28,17 +32,21 @@ def convert_to_markdown(metrics_dict): markdown += "| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A" for run in runs: # add metrics and values - for k, val in metrics_dict[run].items(): - if "best_child_by_primary_metric" in k: - continue - row = f"| {run} | {k} |" - try: - val = float(val) - row += f" {val:.3} |" - except ValueError: - row += f" {val} |" - except TypeError: - row += f" {val} |" + try: + for k, val in metrics_dict[run].items(): + if "best_child_by_primary_metric" in k: + continue + row = f"| {run} | {k} |" + try: + val = float(val) + row += f" {val:.3} |" + except ValueError: + row += f" {val} |" + except TypeError: + row += f" {val} |" + markdown += row + "%0A" + except AttributeError: + row = f"| {run} | {metrics_dict[run]} |" markdown += row + "%0A" return markdown diff --git a/tests/objects.py b/tests/objects.py index 8f1e825..e3223a3 100644 --- a/tests/objects.py +++ b/tests/objects.py @@ -1,4 +1,4 @@ -markdown_conversion_input = { +markdown_conversion_input_1 = { "5815b172-61fe-4e8d-baa2-e4ffe33e77d3": { "testkey": "testvalue" }, @@ -28,4 +28,72 @@ } } -markdown_conversion_output = """## Run Details:%0A%0A Experiment/Pipeline ID: 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 | testkey | testvalue |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_0/TimeSeries comparison_1586189163.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_1/TimeSeries comparison_1586189032.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_2/TimeSeries comparison_1586189099.png |%0A""" +markdown_conversion_output_1 = """## Run Details:%0A%0A Experiment/Pipeline ID: 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| 5815b172-61fe-4e8d-baa2-e4ffe33e77d3 | testkey | testvalue |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_0 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_0/TimeSeries comparison_1586189163.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_1 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_1/TimeSeries comparison_1586189032.png |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | mse | 0.025 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | auc | 0.639 |%0A| HD_7034e363-3172-41e6-8423-eb292f39e233_2 | TimeSeries comparison | aml://artifactId/ExperimentRun/dcid.HD_7034e363-3172-41e6-8423-eb292f39e233_2/TimeSeries comparison_1586189099.png |%0A""" + +markdown_conversion_input_2 = { + "Training samples": 353.0, + "Test samples": 89.0, + "alpha": 0.03, + "mse": 3424.9003158960168 +} + +markdown_conversion_output_2 = """## Run Details:%0A%0A Experiment/Pipeline ID: Training samples %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| Training samples | 353.0 |%0A| Test samples | 89.0 |%0A| alpha | 0.03 |%0A| mse | 3424.9003158960168 |%0A""" + +markdown_conversion_input_3 = { + "HD_9cbf07dc-db35-4700-adc3-92253a378992": { + "best_child_by_primary_metric": { + "metric_name": [ + "mse", + "mse", + "mse" + ], + "timestamp": [ + "2020-03-30 07: 58: 40.110108+00: 00", + "2020-03-30 07: 59: 42.417631+00: 00", + "2020-03-30 07: 59: 42.417631+00: 00" + ], + "run_id": [ + "HD_9cbf07dc-db35-4700-adc3-92253a378992_2", + "HD_9cbf07dc-db35-4700-adc3-92253a378992_1", + "HD_9cbf07dc-db35-4700-adc3-92253a378992_1" + ], + "metric_value": [ + 0.4896792910300023, + 0.33284572706891424, + 0.33284572706891424 + ], + "final": [ + False, + False, + True + ] + } + }, + "HD_9cbf07dc-db35-4700-adc3-92253a378992_0": { + "AUC": 0.6562945178135493, + "RMSE": 0.05120842417634075, + "pAUC": 0.2812047761829226, + "mse": 0.9367719704449606, + "TimeSeries comparison": "aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_0/TimeSeries comparison_1585555145.png" + }, + "HD_9cbf07dc-db35-4700-adc3-92253a378992_1": { + "AUC": 0.3573103979675637, + "pAUC": 0.5616569664904285, + "RMSE": 0.6161010679565609, + "mse": 0.33284572706891424, + "TimeSeries comparison": "aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_1/TimeSeries comparison_1585555146.png" + }, + "HD_9cbf07dc-db35-4700-adc3-92253a378992_2": { + "AUC": 0.1200601984180878, + "pAUC": 0.5805687333099157, + "RMSE": 0.09691275977364577, + "mse": 0.4896792910300023, + "TimeSeries comparison": "aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_2/TimeSeries comparison_1585555076.png" + } +} + +markdown_conversion_output_3 = """## Run Details:%0A%0A Experiment/Pipeline ID: HD_9cbf07dc-db35-4700-adc3-92253a378992 %0A%0A| Run ID | Parameter | Value |%0A| ----- | ----- | ----- |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | AUC | 0.656 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | RMSE | 0.0512 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | pAUC | 0.281 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | mse | 0.937 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_0 | TimeSeries comparison | aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_0/TimeSeries comparison_1585555145.png |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | AUC | 0.357 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | pAUC | 0.562 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | RMSE | 0.616 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | mse | 0.333 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_1 | TimeSeries comparison | aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_1/TimeSeries comparison_1585555146.png |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | AUC | 0.12 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | pAUC | 0.581 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | RMSE | 0.0969 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | mse | 0.49 |%0A| HD_9cbf07dc-db35-4700-adc3-92253a378992_2 | TimeSeries comparison | aml: //artifactId/ExperimentRun/dcid.HD_9cbf07dc-db35-4700-adc3-92253a378992_2/TimeSeries comparison_1585555076.png |%0A""" + +markdown_conversion_input_4 = {} + +markdown_conversion_output_4 = "No metrics returned" diff --git a/tests/test_utils.py b/tests/test_utils.py index 87e2b85..45377a7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,17 +7,36 @@ from utils import AMLConfigurationException, convert_to_markdown, validate_json, load_pipeline_yaml, load_runconfig_yaml, load_runconfig_python from schemas import azure_credentials_schema -from objects import markdown_conversion_input, markdown_conversion_output +from objects import markdown_conversion_input_1, markdown_conversion_output_1, markdown_conversion_input_2, markdown_conversion_output_2, markdown_conversion_input_3, markdown_conversion_output_3, markdown_conversion_input_4, markdown_conversion_output_4 def test_markdown_single_experiment_conversion(): """ Unit test to check the markdown conversion """ - markdown = convert_to_markdown( - markdown_conversion_input + markdown_1 = convert_to_markdown( + metrics_dict=markdown_conversion_input_1 ) - assert markdown == markdown_conversion_output + print(markdown_1) + assert markdown_1 == markdown_conversion_output_1 + + markdown_2 = convert_to_markdown( + metrics_dict=markdown_conversion_input_2 + ) + print(markdown_2) + assert markdown_2 == markdown_conversion_output_2 + + markdown_3 = convert_to_markdown( + metrics_dict=markdown_conversion_input_3 + ) + print(markdown_3) + assert markdown_3 == markdown_conversion_output_3 + + markdown_4 = convert_to_markdown( + metrics_dict=markdown_conversion_input_4 + ) + print(markdown_4) + assert markdown_4 == markdown_conversion_output_4 def test_validate_json_valid_inputs():