From bdc36546391d887c8bb35764a55e8508dd428ec0 Mon Sep 17 00:00:00 2001 From: "pull[bot]" <39814207+pull[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 07:28:01 +0000 Subject: [PATCH] [pull] main from kubeflow:main (#100) * build(deps): bump github.com/docker/docker from 27.1.0+incompatible to 27.1.1+incompatible (#225) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 27.1.0+incompatible to 27.1.1+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v27.1.0...v27.1.1) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: onboard CSI on dependabot (#218) Signed-off-by: Matteo Mortari * build(deps-dev): bump pytest from 8.3.1 to 8.3.2 in /clients/python (#227) Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.1 to 8.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.1...8.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump huggingface-hub from 0.24.0 to 0.24.3 in /clients/python (#228) Bumps [huggingface-hub](https://github.com/huggingface/huggingface_hub) from 0.24.0 to 0.24.3. - [Release notes](https://github.com/huggingface/huggingface_hub/releases) - [Commits](https://github.com/huggingface/huggingface_hub/compare/v0.24.0...v0.24.3) --- updated-dependencies: - dependency-name: huggingface-hub dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps-dev): bump ruff from 0.5.4 to 0.5.5 in /clients/python (#229) Bumps [ruff](https://github.com/astral-sh/ruff) from 0.5.4 to 0.5.5. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.5.4...0.5.5) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * test: show s3_uri_from format in pure REST test (#219) Signed-off-by: Matteo Mortari --------- Signed-off-by: dependabot[bot] Signed-off-by: Matteo Mortari Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Matteo Mortari --- .github/dependabot.yml | 3 +- clients/python/poetry.lock | 50 +++++++++++++++++----------------- go.mod | 2 +- go.sum | 4 +-- test/robot/MRkeywords.resource | 6 ++++ test/robot/ModelRegistry.py | 7 +++++ test/robot/UserStory.robot | 29 ++++++++++++++++++++ 7 files changed, 72 insertions(+), 29 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5176165d..f096679e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,7 @@ updates: directories: - "/" - "/clients/ui/bff/" + - "/csi" schedule: interval: "weekly" - package-ecosystem: "pip" @@ -26,4 +27,4 @@ updates: - package-ecosystem: "npm" directory: "/clients/ui/frontend/" schedule: - interval: "weekly" \ No newline at end of file + interval: "weekly" diff --git a/clients/python/poetry.lock b/clients/python/poetry.lock index 6d0d30df..8e810bd3 100644 --- a/clients/python/poetry.lock +++ b/clients/python/poetry.lock @@ -685,13 +685,13 @@ files = [ [[package]] name = "huggingface-hub" -version = "0.24.0" +version = "0.24.3" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = true python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.24.0-py3-none-any.whl", hash = "sha256:7ad92edefb93d8145c061f6df8d99df2ff85f8379ba5fac8a95aca0642afa5d7"}, - {file = "huggingface_hub-0.24.0.tar.gz", hash = "sha256:6c7092736b577d89d57b3cdfea026f1b0dc2234ae783fa0d59caf1bf7d52dfa7"}, + {file = "huggingface_hub-0.24.3-py3-none-any.whl", hash = "sha256:69ecce486dd6cdad69937ba76779e893c224a670a9d947636c1d5cbd049e44d8"}, + {file = "huggingface_hub-0.24.3.tar.gz", hash = "sha256:bfdc05cc9b64a0e24e8614a44222698799183268f6b68be209aa2df70cff2cde"}, ] [package.dependencies] @@ -1317,13 +1317,13 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pytest" -version = "8.3.1" +version = "8.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.1-py3-none-any.whl", hash = "sha256:e9600ccf4f563976e2c99fa02c7624ab938296551f280835ee6516df8bc4ae8c"}, - {file = "pytest-8.3.1.tar.gz", hash = "sha256:7e8e5c5abd6e93cb1cc151f23e57adc31fcf8cfd2a3ff2da63e23f732de35db6"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [package.dependencies] @@ -1470,29 +1470,29 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruff" -version = "0.5.4" +version = "0.5.5" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.4-py3-none-linux_armv6l.whl", hash = "sha256:82acef724fc639699b4d3177ed5cc14c2a5aacd92edd578a9e846d5b5ec18ddf"}, - {file = "ruff-0.5.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:da62e87637c8838b325e65beee485f71eb36202ce8e3cdbc24b9fcb8b99a37be"}, - {file = "ruff-0.5.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e98ad088edfe2f3b85a925ee96da652028f093d6b9b56b76fc242d8abb8e2059"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c55efbecc3152d614cfe6c2247a3054cfe358cefbf794f8c79c8575456efe19"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9b85eaa1f653abd0a70603b8b7008d9e00c9fa1bbd0bf40dad3f0c0bdd06793"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf497a47751be8c883059c4613ba2f50dd06ec672692de2811f039432875278"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:09c14ed6a72af9ccc8d2e313d7acf7037f0faff43cde4b507e66f14e812e37f7"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f6b8f97b8bad2490240aa84f3e68f390e13fabc9af5c0d3b96b485921cd60"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3520a00c0563d7a7a7c324ad7e2cde2355733dafa9592c671fb2e9e3cd8194c1"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93789f14ca2244fb91ed481456f6d0bb8af1f75a330e133b67d08f06ad85b516"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:029454e2824eafa25b9df46882f7f7844d36fd8ce51c1b7f6d97e2615a57bbcc"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9492320eed573a13a0bc09a2957f17aa733fff9ce5bf00e66e6d4a88ec33813f"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6e1f62a92c645e2919b65c02e79d1f61e78a58eddaebca6c23659e7c7cb4ac7"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:768fa9208df2bec4b2ce61dbc7c2ddd6b1be9fb48f1f8d3b78b3332c7d71c1ff"}, - {file = "ruff-0.5.4-py3-none-win32.whl", hash = "sha256:e1e7393e9c56128e870b233c82ceb42164966f25b30f68acbb24ed69ce9c3a4e"}, - {file = "ruff-0.5.4-py3-none-win_amd64.whl", hash = "sha256:58b54459221fd3f661a7329f177f091eb35cf7a603f01d9eb3eb11cc348d38c4"}, - {file = "ruff-0.5.4-py3-none-win_arm64.whl", hash = "sha256:bd53da65f1085fb5b307c38fd3c0829e76acf7b2a912d8d79cadcdb4875c1eb7"}, - {file = "ruff-0.5.4.tar.gz", hash = "sha256:2795726d5f71c4f4e70653273d1c23a8182f07dd8e48c12de5d867bfb7557eed"}, + {file = "ruff-0.5.5-py3-none-linux_armv6l.whl", hash = "sha256:605d589ec35d1da9213a9d4d7e7a9c761d90bba78fc8790d1c5e65026c1b9eaf"}, + {file = "ruff-0.5.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00817603822a3e42b80f7c3298c8269e09f889ee94640cd1fc7f9329788d7bf8"}, + {file = "ruff-0.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:187a60f555e9f865a2ff2c6984b9afeffa7158ba6e1eab56cb830404c942b0f3"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe26fc46fa8c6e0ae3f47ddccfbb136253c831c3289bba044befe68f467bfb16"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad25dd9c5faac95c8e9efb13e15803cd8bbf7f4600645a60ffe17c73f60779b"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f70737c157d7edf749bcb952d13854e8f745cec695a01bdc6e29c29c288fc36e"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:cfd7de17cef6ab559e9f5ab859f0d3296393bc78f69030967ca4d87a541b97a0"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09b43e02f76ac0145f86a08e045e2ea452066f7ba064fd6b0cdccb486f7c3e7"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0b856cb19c60cd40198be5d8d4b556228e3dcd545b4f423d1ad812bfdca5884"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3687d002f911e8a5faf977e619a034d159a8373514a587249cc00f211c67a091"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ac9dc814e510436e30d0ba535f435a7f3dc97f895f844f5b3f347ec8c228a523"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:af9bdf6c389b5add40d89b201425b531e0a5cceb3cfdcc69f04d3d531c6be74f"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d40a8533ed545390ef8315b8e25c4bb85739b90bd0f3fe1280a29ae364cc55d8"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cab904683bf9e2ecbbe9ff235bfe056f0eba754d0168ad5407832928d579e7ab"}, + {file = "ruff-0.5.5-py3-none-win32.whl", hash = "sha256:696f18463b47a94575db635ebb4c178188645636f05e934fdf361b74edf1bb2d"}, + {file = "ruff-0.5.5-py3-none-win_amd64.whl", hash = "sha256:50f36d77f52d4c9c2f1361ccbfbd09099a1b2ea5d2b2222c586ab08885cf3445"}, + {file = "ruff-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3191317d967af701f1b73a31ed5788795936e423b7acce82a2b63e26eb3e89d6"}, + {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"}, ] [[package]] diff --git a/go.mod b/go.mod index 15a58ac9..d2514fd4 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/containerd/log v0.1.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/docker/docker v27.1.0+incompatible + github.com/docker/docker v27.1.1+incompatible github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect diff --git a/go.sum b/go.sum index f32ca570..b1ea37f3 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs= -github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= diff --git a/test/robot/MRkeywords.resource b/test/robot/MRkeywords.resource index b52afdf1..65f223cb 100644 --- a/test/robot/MRkeywords.resource +++ b/test/robot/MRkeywords.resource @@ -207,3 +207,9 @@ I get ArtifactsByModelVersionID Fail Not Implemented END RETURN ${result} + + +I use util s3_uri_from + [Arguments] ${path} ${bucket} ${endpoint} ${region} + ${result}= s3_uri_from path=${path} bucket=${bucket} endpoint=${endpoint} region=${region} + RETURN ${result} diff --git a/test/robot/ModelRegistry.py b/test/robot/ModelRegistry.py index e8aabd17..60fcbb69 100644 --- a/test/robot/ModelRegistry.py +++ b/test/robot/ModelRegistry.py @@ -1,6 +1,7 @@ from model_registry.core import ModelRegistryAPIClient from model_registry.types import ModelArtifact, ModelVersion, RegisteredModel from robot.libraries.BuiltIn import BuiltIn +import model_registry.utils def write_to_console(s): @@ -33,6 +34,12 @@ async def upsert_model_artifact( write_to_console(p) return (await self.api.upsert_model_artifact(p, model_version_id)).id + def s3_uri_from(self, path, bucket, endpoint, region) -> str: + """ + Expose util to RobotFramework + """ + return model_registry.utils.s3_uri_from(path=path, bucket=bucket, endpoint=endpoint, region=region) + async def test(): demo_instance = ModelRegistry() diff --git a/test/robot/UserStory.robot b/test/robot/UserStory.robot index cdc61774..1376c49b 100644 --- a/test/robot/UserStory.robot +++ b/test/robot/UserStory.robot @@ -96,3 +96,32 @@ As a MLOps engineer I would like to store an owner for the RegisteredModel ${r} Then I get RegisteredModelByID id=${rId} And Should be equal ${r["description"]} Lorem ipsum dolor sit amet And Should be equal ${r["owner"]} My owner + +As a MLOps engineer I want to track a Model from an S3 bucket Data Connection + ${data_connection_name} Set Variable aws-connection-my-data-connection + ${s3_bucket} Set Variable my-bucket + ${s3_path} Set Variable my-path + ${s3_endpoint} Set Variable https://minio-api.acme.org + ${s3_region} Set Variable us-east-1 + + Set To Dictionary ${registered_model} description=The Model name=${name} + Set To Dictionary ${model_version} description=The Version name=v1.0 + ${art_uri} I use util s3_uri_from path=${s3_path} bucket=${s3_bucket} endpoint=${s3_endpoint} region=${s3_region} + Set To Dictionary ${model_artifact} description=The Artifact uri=${art_uri} modelFormatName=onnx modelFormatVersion=1 storageKey=${data_connection_name} storagePath=${s3_path} + Log to console ${model_artifact} + + ${rId} Given I create a RegisteredModel payload=${registered_model} + ${vId} And I create a child ModelVersion registeredModelID=${rId} payload=&{model_version} + ${aId} And I create a child ModelArtifact modelversionId=${vId} payload=&{model_artifact} + ${r} Then I get RegisteredModelByID id=${rId} + And Should be equal ${r["description"]} The Model + ${r} Then I get ModelVersionByID id=${vId} + And Should be equal ${r["description"]} The Version + And Should be equal ${r["name"]} v1.0 + ${r} Then I get ModelArtifactByID id=${aId} + And Should be equal ${r["description"]} The Artifact + And Should be equal ${r["uri"]} s3://my-bucket/my-path?endpoint=https://minio-api.acme.org&defaultRegion=us-east-1 + And Should be equal ${r["modelFormatName"]} onnx + And Should be equal ${r["modelFormatVersion"]} 1 + And Should be equal ${r["storageKey"]} aws-connection-my-data-connection + And Should be equal ${r["storagePath"]} my-path