Merge branch 'main' into dependabot/go_modules/github-com-aws-aws-sdk…

…-go-v2-6bbfd9d636

docs/example/minimal_fasttrackml.py

@@ -1,10 +1,9 @@
 import time
 from random import randint, random
 
-from fasttrackml.entities.metric import Metric
-
 import fasttrackml
 from fasttrackml import FasttrackmlClient
+from fasttrackml.entities.metric import Metric
 
 
 def print_metric_info(history):

go.mod

@@ -8,7 +8,7 @@ replace (
 )
 
 require (
-	cloud.google.com/go/storage v1.42.0
+	cloud.google.com/go/storage v1.43.0
 	dagger.io/dagger v0.11.6
 	dario.cat/mergo v1.0.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
@@ -22,7 +22,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.10.0
 	github.com/go-python/gpython v0.2.0
 	github.com/gofiber/fiber/v2 v2.52.5
-	github.com/gofiber/template/html/v2 v2.1.1
+	github.com/gofiber/template/html/v2 v2.1.2
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/hetiansu5/urlquery v1.2.7

go.sum

@@ -11,8 +11,8 @@ cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0=
 cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE=
 cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU=
 cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng=
-cloud.google.com/go/storage v1.42.0 h1:4QtGpplCVt1wz6g5o1ifXd656P5z+yNgzdw1tVfp0cU=
-cloud.google.com/go/storage v1.42.0/go.mod h1:HjMXRFq65pGKFn6hxj6x3HCyR41uSB72Z0SO/Vn6JFQ=
+cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
+cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
 dagger.io/dagger v0.11.6 h1:aGmMpPETphE091FQ4U4LRxYLxbvLZd5Icm/V1Vv8IOQ=
 dagger.io/dagger v0.11.6/go.mod h1:7JewnFDOKZdJNQbQsPvjemONAn6v3ihuTL1bxBSQG8g=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@@ -118,8 +118,8 @@ github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yG
 github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/gofiber/template v1.8.3 h1:hzHdvMwMo/T2kouz2pPCA0zGiLCeMnoGsQZBTSYgZxc=
 github.com/gofiber/template v1.8.3/go.mod h1:bs/2n0pSNPOkRa5VJ8zTIvedcI/lEYxzV3+YPXdBvq8=
-github.com/gofiber/template/html/v2 v2.1.1 h1:QEy3O3EBkvwDthy5bXVGUseOyO6ldJoiDxlF4+MJiV8=
-github.com/gofiber/template/html/v2 v2.1.1/go.mod h1:2G0GHHOUx70C1LDncoBpe4T6maQbNa4x1CVNFW0wju0=
+github.com/gofiber/template/html/v2 v2.1.2 h1:wkK/mYJ3nIhongTkG3t0QgV4ADdgOYJYVSAF2AHnh8Y=
+github.com/gofiber/template/html/v2 v2.1.2/go.mod h1:E98Z/FzvpaSib06aWEgYk6GXNf3ctoyaJH8yW5ay5ak=
 github.com/gofiber/utils v1.1.0 h1:vdEBpn7AzIUJRhe+CiTOJdUcTg4Q9RK+pEa0KPbLdrM=
 github.com/gofiber/utils v1.1.0/go.mod h1:poZpsnhBykfnY1Mc0KeEa6mSHrS3dV0+oBWyeQmb2e0=
 github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=

pkg/api/mlflow/services/run/converters.go

@@ -3,6 +3,8 @@ package run
 import (
 	"fmt"
 
+	"github.com/google/uuid"
+
 	"github.com/G-Research/fasttrackml/pkg/api/mlflow/api/request"
 	"github.com/G-Research/fasttrackml/pkg/api/mlflow/dao/models"
 )
@@ -29,6 +31,7 @@ func ConvertCreateRunArtifactRequestToModel(
 	namespaceID uint, req *request.LogArtifactRequest,
 ) *models.Artifact {
 	return &models.Artifact{
+		ID:      uuid.New(),
 		Iter:    req.Iter,
 		Step:    req.Step,
 		RunID:   req.RunID,

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

python/client_example.py

@@ -1,10 +1,9 @@
 import time
 from random import randint, random
 
-from fasttrackml.entities.metric import Metric
-
 import fasttrackml
 from fasttrackml import FasttrackmlClient
+from fasttrackml.entities.metric import Metric
 
 
 def print_metric_info(history):

python/client_test.py

@@ -1,14 +1,15 @@
 import os
+import posixpath
 import socket
 import subprocess
 import time
 import uuid
 from random import random, uniform
 
 import pytest
-from fasttrackml.entities import Metric, Param
 
 from fasttrackml import FasttrackmlClient
+from fasttrackml.entities import Metric, Param
 
 LOCALHOST = "127.0.0.1"
 
@@ -115,3 +116,12 @@ def test_init_output_logging(client, server, run):
     for i in range(100):
         log_data = str(uuid.uuid4()) + "\n" + str(uuid.uuid4())
         print(log_data)
+
+
+def test_log_image(client, server, run):
+    # test logging some images
+    for i in range(100):
+        img_local = posixpath.join(os.path.dirname(__file__), "dice.png")
+        assert (
+            client.log_image(run.info.run_id, img_local, "images", "These are dice", 0, 640, 480, "png", i, 0) == None
+        )

python/fasttrackml/_tracking_service/client.py

@@ -144,3 +144,21 @@ def log_output(
         data: str,
     ):
         self.custom_store.log_output(run_id, data)
+
+    def log_image(
+        self,
+        run_id: str,
+        filename: str,
+        artifact_path: str,
+        caption: str,
+        index: int,
+        width: int,
+        height: int,
+        format: str,
+        step: int,
+        iter: int,
+    ):
+        # 1. log the artifact
+        self.log_artifact(run_id, filename, artifact_path)
+        # 2. log the image metadata
+        self.custom_store.log_image(run_id, filename, artifact_path, caption, index, width, height, format, step, iter)

python/fasttrackml/client.py

@@ -461,3 +461,56 @@ def log_output(
             client.set_terminated(run.info.run_id)
         """
         self._tracking_client.log_output(run_id, data)
+
+    def log_image(
+        self,
+        run_id: str,
+        filename: str,
+        artifact_path: str,
+        caption: str,
+        index: int,
+        width: int,
+        height: int,
+        format: str,
+        step: int,
+        iter: int,
+    ) -> None:
+        """
+        Log an image artifact for the provided run which will be viewable in the Images explorer.
+        The image itself will be stored in the configured artifact store (S3-compatible or local).
+
+        Args:
+            run_id: String ID of the run
+            filename: The filename of the image in the local filesystem
+            artifact_path: The optional path to append to the artifact_uri
+            caption: The image caption
+            index: The image index
+            width: The image width
+            height: The image height
+            format: The image format
+            step: The image step
+            iter: The image iteration
+
+        .. code-block:: python
+            :caption: Example
+
+            from fasttrackml import FasttrackmlClient
+
+            # Create a run under the default experiment (whose id is '0').
+            # Since these are low-level CRUD operations, this method will create a run.
+            # To end the run, you'll have to explicitly end it.
+            client = FasttrackmlClient()
+            experiment_id = "0"
+            run = client.create_run(experiment_id)
+            print_run_info(run)
+            print("--")
+
+            # Log an image
+            for step in range(10):
+                filename = generate_image(step) # some function that generates an image
+                client.log_image(run.info.run_id, filename, "This is an image", "images", step, 100, 100, "png", step, 0)
+            client.set_terminated(run.info.run_id)
+        """
+        return self._tracking_client.log_image(
+            run_id, filename, artifact_path, caption, index, width, height, format, step, iter
+        )

python/fasttrackml/store/custom_rest_store.py

@@ -1,4 +1,6 @@
 import json
+import os
+import posixpath
 import threading
 from typing import Dict, Optional, Sequence
 
@@ -274,3 +276,45 @@ def log_output(self, run_id, data):
                 error_code=result["error_code"],
             )
         return result
+
+    def log_image(
+        self,
+        run_id: str,
+        filename: str,
+        artifact_path: str,
+        caption: str,
+        index: int,
+        width: int,
+        height: int,
+        format: str,
+        step: int,
+        iter: int,
+    ):
+        storage_path = posixpath.join(artifact_path, os.path.basename(filename))
+        request_body = {
+            "run_id": run_id,
+            "blob_uri": storage_path,
+            "caption": caption,
+            "index": index,
+            "width": width,
+            "height": height,
+            "format": format,
+            "step": step,
+            "iter": iter,
+        }
+        result = http_request(
+            **{
+                "host_creds": self.get_host_creds(),
+                "endpoint": "/api/2.0/mlflow/runs/log-artifact",
+                "method": "POST",
+                "json": request_body,
+            }
+        )
+        if result.status_code != 201:
+            result = result.json()
+        if "error_code" in result:
+            raise MlflowException(
+                message=result["message"],
+                error_code=result["error_code"],
+            )
+        return result