From 3de4acddcf9c002774a443a20cde0b7ab1cb5d08 Mon Sep 17 00:00:00 2001 From: Markus Petke Date: Wed, 13 Sep 2023 09:15:02 +0200 Subject: [PATCH] fix review findings --- .devcontainer/scripts/post-create.sh | 41 ++---- .gitignore | 1 + README.md | 2 +- .../src/gen_desired_state.py | 122 +++++++++--------- desired_state_generator/src/install_deps.py | 40 ------ desired_state_generator/src/requirements.txt | 1 - desired_state_generator/test/requirements.txt | 1 + .../test/test_gen_desired_state.py | 27 ++-- manifest.json | 26 +++- runtime_k3d/src/install_deps.py | 40 ------ runtime_kanto/src/install_deps.py | 40 ------ runtime_local/src/install_deps.py | 40 ------ 12 files changed, 108 insertions(+), 273 deletions(-) mode change 100644 => 100755 .devcontainer/scripts/post-create.sh delete mode 100644 desired_state_generator/src/install_deps.py create mode 100644 desired_state_generator/test/requirements.txt delete mode 100644 runtime_k3d/src/install_deps.py delete mode 100644 runtime_kanto/src/install_deps.py delete mode 100644 runtime_local/src/install_deps.py diff --git a/.devcontainer/scripts/post-create.sh b/.devcontainer/scripts/post-create.sh old mode 100644 new mode 100755 index 50c2d4e6..cc51592f --- a/.devcontainer/scripts/post-create.sh +++ b/.devcontainer/scripts/post-create.sh @@ -18,36 +18,15 @@ echo "### Install python requirements ###" echo "#######################################################" # Update pip before installing requirements pip3 install --upgrade pip -REQUIREMENTS="runtime_k3d/src/requirements.txt" -if [ -f $REQUIREMENTS ]; then - pip3 install -r $REQUIREMENTS -fi -REQUIREMENTS="runtime_kanto/src/requirements.txt" -if [ -f $REQUIREMENTS ]; then - pip3 install -r $REQUIREMENTS -fi -# Dependencies for the app -REQUIREMENTS="runtime_local/src/requirements.txt" -if [ -f $REQUIREMENTS ]; then - pip3 install -r $REQUIREMENTS -fi +for package in runtime_k3d runtime_kanto runtime_local desired_state_generator; do + REQUIREMENTS="$package/src/requirements.txt" + if [ -f $REQUIREMENTS ]; then + pip3 install -r $REQUIREMENTS + fi + REQUIREMENTS="$package/test/requirements.txt" + if [ -f $REQUIREMENTS ]; then + pip3 install -r $REQUIREMENTS + fi -echo "#######################################################" -echo "### Install test requirements ###" -echo "#######################################################" -# Update pip before installing requirements -pip3 install --upgrade pip -REQUIREMENTS="runtime_k3d/test/requirements.txt" -if [ -f $REQUIREMENTS ]; then - pip3 install -r $REQUIREMENTS -fi -REQUIREMENTS="runtime_kanto/test/requirements.txt" -if [ -f $REQUIREMENTS ]; then - pip3 install -r $REQUIREMENTS -fi -# Dependencies for the app -REQUIREMENTS="runtime_local/test/requirements.txt" -if [ -f $REQUIREMENTS ]; then - pip3 install -r $REQUIREMENTS -fi +done diff --git a/.gitignore b/.gitignore index 5aaf1ca9..92e4f840 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build logs results .coverage +tmp diff --git a/README.md b/README.md index 25efbeac..010a4a72 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Function | Description :---|:--- `$pathInWorkspaceOrPackage( )` | Resolves a path dynamically either to the local project workspace, if the file is available or falls back to a file in the package repository. If none of these files is available an exception is raised. -## Generators +## Deployment spec generators This package contains the following generators: diff --git a/desired_state_generator/src/gen_desired_state.py b/desired_state_generator/src/gen_desired_state.py index 18be685b..5d446fc7 100644 --- a/desired_state_generator/src/gen_desired_state.py +++ b/desired_state_generator/src/gen_desired_state.py @@ -16,12 +16,22 @@ import hashlib import json import re -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional -import requests import velocitas_lib import velocitas_lib.services +VSS_SOURCE_DEFAULT = "vss-source-default-vss" +VSS_SOURCE_CUSTOM = "vss-source-custom-vss" +DATABROKER_ID = "data-broker-grpc" +GRPC_INTERFACE_ID = "dataprovider-proto-grpc" +DATABROKER_PACKAGE_ID = "vehicledatabroker" +MQTT_PACKAGE_ID = "mqtt-broker" + +VSS_INTERFACE = "vehicle-signal-interface" +PUBSUB_INTERFACE = "pubsub" +GRPC_INTERFACE = "grpc-interface" + def is_uri(path: str) -> bool: """Check if the provided path is a URI. @@ -36,10 +46,11 @@ def is_uri(path: str) -> bool: def parse_vehicle_signal_interface(config: Dict[str, Any]) -> List[str]: - """Parses the vehicle signal interface config. + """Parse the vehicle signal interface config. Args: - config: The json-config of the interface, as defined in the appManifest.json. + config (Dict[str, Any]): The json-config of the interface, + as defined in the appManifest.json. Returns: List[str]: A list of requirements defined by the config. @@ -52,15 +63,12 @@ def parse_vehicle_signal_interface(config: Dict[str, Any]) -> List[str]: version = "" if vss_release_prefix in src: version = src.removeprefix(vss_release_prefix).split("/")[0] - requirements.append(f"vss-source-default-vss:{version}") - elif is_uri(src): - version = get_md5_from_uri(src) - requirements.append(f"vss-source-custom-vss:{version}") + requirements.append(f"{VSS_SOURCE_DEFAULT}:{version}") else: - version = get_md5_for_file(src) - requirements.append(f"vss-source-custom-vss:{version}") + version = get_md5_from_file_content(src) + requirements.append(f"{VSS_SOURCE_CUSTOM}:{version}") - requirements.append(f"data-broker-grpc:{get_package_version('vehicledatabroker')}") + requirements.append(f"{DATABROKER_ID}:{get_package_version(DATABROKER_PACKAGE_ID)}") datapoints = config["datapoints"]["required"] for datapoint in datapoints: @@ -72,45 +80,34 @@ def parse_vehicle_signal_interface(config: Dict[str, Any]) -> List[str]: def parse_grpc_interface(config: Dict[str, Any]) -> str: - """Parses the grpc interface config. + """Parse the grpc interface config. Args: - config: The json-config of the interface, as defined in the appManifest.json. + config (Dict[str, Any]): The json-config of the interface, + as defined in the appManifest.json. Returns: str: The requirement with md5-hash of the proto-file as version. """ src = str(config["src"]) - return f"dataprovider-proto-grpc:{get_md5_from_uri(src)}" + return f"{GRPC_INTERFACE_ID}:{get_md5_from_file_content(src)}" -def get_md5_from_uri(src: str) -> str: - """Get the md5-hash of a file defined by an URI. +def get_md5_from_file_content(src: str) -> str: + """Get the md5-hash of the contents of a file defined by a source. Args: - str: The URI of the file. + src (str): The source of the file. Can either be a local file-path or an URI Returns: str: The md5-hash of the file. """ - md5 = hashlib.md5(usedforsecurity=False) - with requests.get(src, timeout=30) as source: - for chunk in source.iter_content(chunk_size=4096): - md5.update(chunk) - - return md5.hexdigest() + file_path = src + if is_uri(src): + file_path = "./tmp" + velocitas_lib.download_file(src, file_path) - -def get_md5_for_file(file_path: str): - """Get the md5-hash of a local file defined by a path. - - Args: - str: The local path to the file. - - Returns: - str: The md5-hash of the file. - """ md5 = hashlib.md5(usedforsecurity=False) with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): @@ -120,10 +117,13 @@ def get_md5_for_file(file_path: str): def get_package_version(package_id: str) -> str: - """Get the version of the used mqtt-broker. + """Get the version of the provided package. + + Args: + package_id (str): The ID of the package Returns: - str: The version of the mqtt-broker defined in the runtime.json. + str: The version of the package defined in the runtime.json. """ config = velocitas_lib.services.get_specific_service(package_id).config version = config.image.split(":")[-1] @@ -131,10 +131,11 @@ def get_package_version(package_id: str) -> str: def parse_interfaces(interfaces: List[Dict[str, Any]]) -> List[str]: - """Parses the defined interfaces. + """Parse the defined interfaces. Args: - interfaces: The json-array of interfaces, as defined in the appManifest.json. + interfaces (List[Dict[str, Any]])): The json-array of interfaces, + as defined in the appManifest.json. Returns: List[str]: A list of requirements defined by the interface definitions. @@ -142,36 +143,17 @@ def parse_interfaces(interfaces: List[Dict[str, Any]]) -> List[str]: requirements = [] for interface in interfaces: interface_type = interface["type"] - if interface_type == "vehicle-signal-interface": + if interface_type == VSS_INTERFACE: requirements += parse_vehicle_signal_interface(interface["config"]) - elif interface_type == "pubsub": - requirements.append(f"mqtt:{get_package_version('mqtt-broker')}") - elif interface_type == "grpc-interface": + elif interface_type == PUBSUB_INTERFACE: + requirements.append(f"mqtt:{get_package_version(MQTT_PACKAGE_ID)}") + elif interface_type == GRPC_INTERFACE: requirements.append(parse_grpc_interface(interface["config"])) return requirements -def main(): - parser = argparse.ArgumentParser("generate-desired-state") - parser.add_argument( - "-o", - "--output-file-path", - type=str, - required=False, - help="Path to the folder where the manifest should be placed.", - ) - parser.add_argument( - "-s", - "--source", - type=str, - required=True, - help="The URL of the image including the tag.", - ) - args = parser.parse_args() - - output_file_path = args.output_file_path - source = args.source +def main(source: str, output_file_path: Optional[str] = None): imageName = source.split(":")[0].split("/")[-1] version = source.split(":")[1] @@ -202,4 +184,20 @@ def main(): if __name__ == "__main__": - main() + parser = argparse.ArgumentParser("generate-desired-state") + parser.add_argument( + "-o", + "--output-file-path", + type=str, + required=False, + help="Path to the folder where the manifest should be placed.", + ) + parser.add_argument( + "-s", + "--source", + type=str, + required=True, + help="The URL of the image including the tag.", + ) + args = parser.parse_args() + main(args.source, args.output_file_path) diff --git a/desired_state_generator/src/install_deps.py b/desired_state_generator/src/install_deps.py deleted file mode 100644 index 78417ccb..00000000 --- a/desired_state_generator/src/install_deps.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2023 Robert Bosch GmbH and Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Provides methods and functions to download and install dependencies.""" - -import os -import subprocess -import sys - - -def get_script_path() -> str: - """Return the absolute path to the directory the invoked Python script - is located in.""" - return os.path.dirname(os.path.realpath(sys.argv[0])) - - -def pip(args: list[str]) -> None: - """Invoke the pip process with the given arguments.""" - subprocess.check_call([sys.executable, "-m", "pip", *args]) - - -def install_packages(): - """Install all required Python packages.""" - script_path = get_script_path() - pip(["install", "-r", f"{script_path}/requirements.txt"]) - - -if __name__ == "__main__": - install_packages() diff --git a/desired_state_generator/src/requirements.txt b/desired_state_generator/src/requirements.txt index 9d554d4a..ee428502 100644 --- a/desired_state_generator/src/requirements.txt +++ b/desired_state_generator/src/requirements.txt @@ -1,2 +1 @@ -requests==2.31.0 git+https://github.com/eclipse-velocitas/velocitas-lib.git@v0.0.2 diff --git a/desired_state_generator/test/requirements.txt b/desired_state_generator/test/requirements.txt new file mode 100644 index 00000000..805eb2a9 --- /dev/null +++ b/desired_state_generator/test/requirements.txt @@ -0,0 +1 @@ +pytest==7.2.1 diff --git a/desired_state_generator/test/test_gen_desired_state.py b/desired_state_generator/test/test_gen_desired_state.py index 6996c841..721c5712 100644 --- a/desired_state_generator/test/test_gen_desired_state.py +++ b/desired_state_generator/test/test_gen_desired_state.py @@ -17,20 +17,21 @@ import sys from pathlib import Path -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) -from gen_desired_state import get_md5_for_file, get_md5_from_uri, is_uri - +import pytest -def test_get_md5_for_file(): - hash = get_md5_for_file(f"{Path.cwd()}/LICENSE") - # generated with https://emn178.github.io/online-tools/md5_checksum.html - assert hash == "86d3f3a95c324c9479bd8986968f4327" - - -def test_get_md5_for_uri(): - hash = get_md5_from_uri( - "https://raw.githubusercontent.com/eclipse-velocitas/devenv-runtimes/main/LICENSE" - ) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) +from gen_desired_state import get_md5_from_file_content, is_uri + + +@pytest.mark.parametrize( + "src", + [ + f"{Path.cwd()}/LICENSE", + "https://raw.githubusercontent.com/eclipse-velocitas/devenv-runtimes/main/LICENSE", + ], +) +def test_get_md5_for_file(src): + hash = get_md5_from_file_content(src) # generated with https://emn178.github.io/online-tools/md5_checksum.html assert hash == "86d3f3a95c324c9479bd8986968f4327" diff --git a/manifest.json b/manifest.json index a6a84dd2..d8848ef8 100644 --- a/manifest.json +++ b/manifest.json @@ -27,7 +27,11 @@ "id": "install-deps", "executable": "python3", "args": [ - "./runtime_local/src/install_deps.py" + "-m", + "pip", + "install", + "-r", + "./runtime_local/src/requirements.txt" ] }, { @@ -89,7 +93,11 @@ "id": "install-deps", "executable": "python3", "args": [ - "./runtime_k3d/src/install_deps.py" + "-m", + "pip", + "install", + "-r", + "./runtime_k3d/src/requirements.txt" ] }, { @@ -203,7 +211,11 @@ "id": "install-deps", "executable": "python3", "args": [ - "./runtime_kanto/src/install_deps.py" + "-m", + "pip", + "install", + "-r", + "./runtime_kanto/src/requirements.txt" ] }, { @@ -285,13 +297,17 @@ { "id": "pantaris-integration", "alias": "pantaris", - "type": "generator", + "type": "setup", "programs": [ { "id": "install-deps", "executable": "python3", "args": [ - "./desired_state_generator/src/install_deps.py" + "-m", + "pip", + "install", + "-r", + "./desired_state_generator/src/requirements.txt" ] }, { diff --git a/runtime_k3d/src/install_deps.py b/runtime_k3d/src/install_deps.py deleted file mode 100644 index 78417ccb..00000000 --- a/runtime_k3d/src/install_deps.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2023 Robert Bosch GmbH and Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Provides methods and functions to download and install dependencies.""" - -import os -import subprocess -import sys - - -def get_script_path() -> str: - """Return the absolute path to the directory the invoked Python script - is located in.""" - return os.path.dirname(os.path.realpath(sys.argv[0])) - - -def pip(args: list[str]) -> None: - """Invoke the pip process with the given arguments.""" - subprocess.check_call([sys.executable, "-m", "pip", *args]) - - -def install_packages(): - """Install all required Python packages.""" - script_path = get_script_path() - pip(["install", "-r", f"{script_path}/requirements.txt"]) - - -if __name__ == "__main__": - install_packages() diff --git a/runtime_kanto/src/install_deps.py b/runtime_kanto/src/install_deps.py deleted file mode 100644 index 78417ccb..00000000 --- a/runtime_kanto/src/install_deps.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2023 Robert Bosch GmbH and Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Provides methods and functions to download and install dependencies.""" - -import os -import subprocess -import sys - - -def get_script_path() -> str: - """Return the absolute path to the directory the invoked Python script - is located in.""" - return os.path.dirname(os.path.realpath(sys.argv[0])) - - -def pip(args: list[str]) -> None: - """Invoke the pip process with the given arguments.""" - subprocess.check_call([sys.executable, "-m", "pip", *args]) - - -def install_packages(): - """Install all required Python packages.""" - script_path = get_script_path() - pip(["install", "-r", f"{script_path}/requirements.txt"]) - - -if __name__ == "__main__": - install_packages() diff --git a/runtime_local/src/install_deps.py b/runtime_local/src/install_deps.py deleted file mode 100644 index 78417ccb..00000000 --- a/runtime_local/src/install_deps.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2023 Robert Bosch GmbH and Microsoft Corporation -# -# This program and the accompanying materials are made available under the -# terms of the Apache License, Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0. -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Provides methods and functions to download and install dependencies.""" - -import os -import subprocess -import sys - - -def get_script_path() -> str: - """Return the absolute path to the directory the invoked Python script - is located in.""" - return os.path.dirname(os.path.realpath(sys.argv[0])) - - -def pip(args: list[str]) -> None: - """Invoke the pip process with the given arguments.""" - subprocess.check_call([sys.executable, "-m", "pip", *args]) - - -def install_packages(): - """Install all required Python packages.""" - script_path = get_script_path() - pip(["install", "-r", f"{script_path}/requirements.txt"]) - - -if __name__ == "__main__": - install_packages()