Merge sync remote-tracking branch 'upstream/main' into tarilabs-20240…

…912-sync

Signed-off-by: Matteo Mortari <matteo.mortari@gmail.com>

CONTRIBUTING.md

@@ -84,6 +84,24 @@ export TESTCONTAINERS_RYUK_PRIVILEGED=true
 
 when running TestContainer-based Model Registry Python tests (for more information, see [here](https://pypi.org/project/testcontainers/#:~:text=TESTCONTAINERS_RYUK_PRIVILEGED)).
 
+#### Fedora
+
+Fedora requires further setup to make testcontainers work with Podman, the following steps are required:
+
+- start the podman socket service
+
+```sh
+systemctl --user start podman.socket
+```
+
+- set the environment variable
+
+```sh
+export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock
+```
+
+If you need more information, please refer to the [Testcontainers using Podman](https://golang.testcontainers.org/system_requirements/using_podman/).
+
 ### Colima
 
 Colima offers Rosetta (Apple specific) emulation which is handy since the Google MLMD project dependency is x86 specific.

clients/python/README.md

@@ -2,10 +2,37 @@
 
 [![Python](https://img.shields.io/badge/python%20-3.9%7C3.10%7C3.11%7C3.12-blue)](https://github.com/kubeflow/model-registry)
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](../../../LICENSE)
-[Documentation](https://model-registry.readthedocs.io/en/latest/)
+[![Read the Docs](https://img.shields.io/readthedocs/model-registry)](https://model-registry.readthedocs.io/en/latest/)
+[![Tutorial Website](https://img.shields.io/badge/Website-green?style=plastic&label=Tutorial&labelColor=blue)](https://www.kubeflow.org/docs/components/model-registry/getting-started/)
 
 This library provides a high level interface for interacting with a model registry server.
 
+## Installation
+
+In your Python environment, you can install the latest version of the Model Registry Python client with:
+
+```
+pip install --pre model-registry
+```
+
+### Installing extras
+
+Some capabilities of this Model Registry Python client, such as [importing model from Hugging Face](#importing-from-hugging-face-hub),
+require additional dependencies.
+
+By [installing an extra variant](https://packaging.python.org/en/latest/tutorials/installing-packages/#installing-extras) of this package
+the additional dependencies will be managed for you automatically, for instance with:
+
+```
+pip install --pre "model-registry[hf]"
+```
+
+This step is not required if you already installed the additional dependencies already, for instance with:
+
+```
+pip install huggingface-hub
+```
+
 ## Basic usage
 
 ```py
@@ -73,7 +100,9 @@ model = registry.register_model(
 ### Importing from Hugging Face Hub
 
 To import models from Hugging Face Hub, start by installing the `huggingface-hub` package, either directly or as an
-extra (available as `model-registry[hf]`).
+extra (available as `model-registry[hf]`). Reference section "[installing extras](#installing-extras)" above for
+more information.
+
 Models can be imported with
 
 ```py

clients/python/src/model_registry/_client.py

@@ -247,7 +247,17 @@ def register_hf_model(
         try:
             from huggingface_hub import HfApi, hf_hub_url, utils
         except ImportError as e:
-            msg = "huggingface_hub is not installed"
+            msg = """package `huggingface-hub` is not installed.
+            To import models from Hugging Face Hub, start by installing the `huggingface-hub` package, either directly or as an
+            extra (available as `model-registry[hf]`), e.g.:
+            ```sh
+            !pip install --pre model-registry[hf]
+            ```
+            or
+            ```sh
+            !pip install huggingface-hub
+            ```
+            """
             raise StoreError(msg) from e
 
         api = HfApi()

clients/python/tests/test_client.py

@@ -107,6 +107,53 @@ async def test_update_models(client: ModelRegistry):
     assert client.update(ma).description == new_description
 
 
+@pytest.mark.e2e
+async def test_update_logical_model_with_labels(client: ModelRegistry):
+    """As a MLOps engineer I would like to store some labels
+
+    A custom property of type string, with empty string value, shall be considered a Label; this is also semantically compatible for properties having empty string values in general.
+    """
+    name = "test_model"
+    version = "1.0.0"
+    rm = client.register_model(
+        name,
+        "s3",
+        model_format_name="test_format",
+        model_format_version="test_version",
+        version=version,
+    )
+    assert rm.id
+    mv = client.get_model_version(name, version)
+    assert mv.id
+    ma = client.get_model_artifact(name, version)
+    assert ma.id
+
+    rm_labels = {
+        "my-label1": "",
+        "my-label2": "",
+    }
+    rm.custom_properties = rm_labels
+    client.update(rm)
+
+    mv_labels = {
+        "my-label3": "",
+        "my-label4": "",
+    }
+    mv.custom_properties = mv_labels
+    client.update(mv)
+
+    ma_labels = {
+        "my-label5": "",
+        "my-label6": "",
+    }
+    ma.custom_properties = ma_labels
+    client.update(ma)
+
+    assert client.get_registered_model(name).custom_properties == rm_labels
+    assert client.get_model_version(name, version).custom_properties == mv_labels
+    assert client.get_model_artifact(name, version).custom_properties == ma_labels
+
+
 @pytest.mark.e2e
 async def test_update_preserves_model_info(client: ModelRegistry):
     name = "test_model"

clients/ui/bff/Dockerfile

@@ -34,8 +34,4 @@ USER 65532:65532
 # Expose port 4000
 EXPOSE 4000
 
-# Define environment variables
-ENV PORT 4001
-ENV ENV development
-
 ENTRYPOINT ["/bff"]

clients/ui/bff/Makefile

@@ -2,6 +2,7 @@ CONTAINER_TOOL ?= docker
 IMG ?= model-registry-bff:latest
 PORT ?= 4000
 MOCK_K8S_CLIENT ?= false
+MOCK_MR_CLIENT ?= false
 
 .PHONY: all
 all: build
@@ -32,7 +33,7 @@ build: fmt vet test
 
 .PHONY: run
 run: fmt vet
-	go run ./cmd/main.go  --port=$(PORT) --mock-k8s-client=$(MOCK_K8S_CLIENT)
+	go run ./cmd/main.go  --port=$(PORT) --mock-k8s-client=$(MOCK_K8S_CLIENT) --mock-mr-client=$(MOCK_MR_CLIENT)
 
 .PHONY: docker-build
 docker-build:

clients/ui/bff/README.md

@@ -31,9 +31,9 @@ After building it, you can run our app with:
 ```shell
 make run
 ```
-If you want to use a different port or mock kubernetes client, useful for front-end development, you can run:
+If you want to use a different port, mock kubernetes client or model registry client - useful for front-end development, you can run:
 ```shell
-make run PORT=8000 MOCK_K8S_CLIENT=true
+make run PORT=8000 MOCK_K8S_CLIENT=true MOCK_MR_CLIENT=true
 ```
 
 # Building and Deploying

clients/ui/bff/api/app.go

@@ -23,10 +23,11 @@ const (
 )
 
 type App struct {
-	config           config.EnvConfig
-	logger           *slog.Logger
-	models           data.Models
-	kubernetesClient integrations.KubernetesClientInterface
+	config              config.EnvConfig
+	logger              *slog.Logger
+	models              data.Models
+	kubernetesClient    integrations.KubernetesClientInterface
+	modelRegistryClient data.ModelRegistryClientInterface
 }
 
 func NewApp(cfg config.EnvConfig, logger *slog.Logger) (*App, error) {
@@ -43,10 +44,23 @@ func NewApp(cfg config.EnvConfig, logger *slog.Logger) (*App, error) {
 		return nil, fmt.Errorf("failed to create Kubernetes client: %w", err)
 	}
 
+	var mrClient data.ModelRegistryClientInterface
+
+	if cfg.MockMRClient {
+		mrClient, err = mocks.NewModelRegistryClient(logger)
+	} else {
+		mrClient, err = data.NewModelRegistryClient(logger)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create ModelRegistry client: %w", err)
+	}
+
 	app := &App{
-		config:           cfg,
-		logger:           logger,
-		kubernetesClient: k8sClient,
+		config:              cfg,
+		logger:              logger,
+		kubernetesClient:    k8sClient,
+		modelRegistryClient: mrClient,
 	}
 	return app, nil
 }
@@ -59,7 +73,7 @@ func (app *App) Routes() http.Handler {
 
 	// HTTP client routes
 	router.GET(HealthCheckPath, app.HealthcheckHandler)
-	router.GET(RegisteredModelsPath, app.AttachRESTClient(app.GetRegisteredModelsHandler))
+	router.GET(RegisteredModelsPath, app.AttachRESTClient(app.GetAllRegisteredModelsHandler))
 	router.GET(RegisteredModelPath, app.AttachRESTClient(app.GetRegisteredModelHandler))
 	router.POST(RegisteredModelsPath, app.AttachRESTClient(app.CreateRegisteredModelHandler))
 

clients/ui/bff/api/helpers.go

@@ -11,6 +11,8 @@ import (
 
 type Envelope map[string]interface{}
 
+type TypedEnvelope[T any] map[string]T
+
 func (app *App) WriteJSON(w http.ResponseWriter, status int, data any, headers http.Header) error {
 
 	js, err := json.MarshalIndent(data, "", "\t")

clients/ui/bff/api/registered_models_handler.go

@@ -6,28 +6,27 @@ import (
 	"fmt"
 	"github.com/julienschmidt/httprouter"
 	"github.com/kubeflow/model-registry/pkg/openapi"
-	"github.com/kubeflow/model-registry/ui/bff/data"
 	"github.com/kubeflow/model-registry/ui/bff/integrations"
 	"github.com/kubeflow/model-registry/ui/bff/validation"
 	"net/http"
 )
 
-func (app *App) GetRegisteredModelsHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+func (app *App) GetAllRegisteredModelsHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	//TODO (ederign) implement pagination
 	client, ok := r.Context().Value(httpClientKey).(integrations.HTTPClientInterface)
 	if !ok {
 		app.serverErrorResponse(w, r, errors.New("REST client not found"))
 		return
 	}
 
-	modelList, err := data.GetAllRegisteredModels(client)
+	modelList, err := app.modelRegistryClient.GetAllRegisteredModels(client)
 	if err != nil {
 		app.serverErrorResponse(w, r, err)
 		return
 	}
 
 	modelRegistryRes := Envelope{
-		"registered_models": modelList,
+		"registered_model_list": modelList,
 	}
 
 	err = app.WriteJSON(w, http.StatusOK, modelRegistryRes, nil)
@@ -60,7 +59,7 @@ func (app *App) CreateRegisteredModelHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	createdModel, err := data.CreateRegisteredModel(client, jsonData)
+	createdModel, err := app.modelRegistryClient.CreateRegisteredModel(client, jsonData)
 	if err != nil {
 		var httpErr *integrations.HTTPError
 		if errors.As(err, &httpErr) {
@@ -91,13 +90,13 @@ func (app *App) GetRegisteredModelHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	model, err := data.GetRegisteredModel(client, ps.ByName(RegisteredModelId))
+	model, err := app.modelRegistryClient.GetRegisteredModel(client, ps.ByName(RegisteredModelId))
 	if err != nil {
 		app.serverErrorResponse(w, r, err)
 		return
 	}
 
-	if _, ok := model.GetNameOk(); !ok {
+	if _, ok := model.GetIdOk(); !ok {
 		app.notFoundResponse(w, r)
 		return
 	}