diff --git a/.github/actions/update-velocitas-json/action.yml b/.github/actions/update-velocitas-json/action.yml new file mode 100644 index 00000000..35d722f7 --- /dev/null +++ b/.github/actions/update-velocitas-json/action.yml @@ -0,0 +1,43 @@ +# Copyright (c) 2023 Robert Bosch GmbH +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +name: update-velocitas-json +description: Update the project's velocitas JSON. +inputs: + project_path: + description: Path to the directory containing the Velocitas project + required: true + template_path: + description: Path to a JSON file which will replace the .velocitas.json file. + default: None +runs: + using: composite + steps: + - run: | + COMMIT_REF=$GITHUB_SHA + + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; + then + echo "Running in context of a PR!" + COMMIT_REF=$GITHUB_HEAD_REF + fi + + if [ "${{ inputs.template_path }}" != "None" ]; then + echo "Copying template file..." + cp -f ${{ inputs.template_path }} ${{ inputs.project_path }}/.velocitas.json + fi + + NEW_CONFIG="$(jq --arg COMMIT_REF "$COMMIT_REF" '.packages[0].version |= $COMMIT_REF' ${{ inputs.project_path }}/.velocitas.json)" + echo "${NEW_CONFIG}" > ${{ inputs.project_path }}/.velocitas.json + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a6a1754..49ae0e15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install deps run: | @@ -48,26 +48,34 @@ jobs: name: "Run unit tests" runs-on: ubuntu-22.04 container: ghcr.io/eclipse-velocitas/devcontainer-base-images/python:v0.1 + strategy: + matrix: + component: ["setup", "grpc-interface-support", "vehicle-model-lifecycle", "sdk-installer"] + fail-fast: false steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install required packages run: | - pip install -r vehicle-model-lifecycle/src/requirements.txt - pip install -r vehicle-model-lifecycle/test/requirements.txt + if [ -e "${{ matrix.component }}/requirements.txt" ]; then + pip install -r ${{ matrix.component }}/requirements.txt + fi + + if [ -e "${{ matrix.component }}/test/requirements.txt" ]; then + pip install -r ${{ matrix.component }}/test/requirements.txt + fi - name: unit test shell: bash run: | - pytest --rootdir=./vehicle-model-lifecycle/test \ - --override-ini junit_family=xunit1 \ + pytest --override-ini junit_family=xunit1 \ --junit-xml=./results/UnitTest/junit.xml \ --cov . \ --cov-report=xml:results/CodeCoverage/cobertura-coverage.xml \ --cov-branch \ - ./vehicle-model-lifecycle/test + ./${{ matrix.component }}/test/unit - name: Publish Test Report uses: mikepenz/action-junit-report@v3 @@ -93,96 +101,50 @@ jobs: strategy: matrix: language: ["python", "cpp"] + component: ["setup", "grpc-interface-support", "vehicle-model-lifecycle", "sdk-installer"] fail-fast: false steps: - name: Checkout repository - uses: actions/checkout@v3 - - name: Checkout template repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - repository: eclipse-velocitas/vehicle-app-${{ matrix.language }}-template - path: test/${{ matrix.language }}/repo - - - name: Install requirements - run: pip install -r test/requirements.txt + fetch-depth: 0 - - name: Prepare .velocitas.json for integration test + - name: Add safe directory run: | - COMMIT_REF=$GITHUB_SHA - - if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; - then - echo "Running in context of a PR!" - COMMIT_REF=$GITHUB_HEAD_REF - fi - - NEW_CONFIG="$(jq --arg COMMIT_REF "$COMMIT_REF" '.packages[0].version |= $COMMIT_REF' test/${{ matrix.language }}/.velocitas.json)" - echo "${NEW_CONFIG}" > test/${{ matrix.language }}/repo/.velocitas.json + git config --global --add safe.directory $(pwd) - - name: Init velocitas project + - name: Get template repo ref + id: get-repo-ref run: | - cd test/${{ matrix.language }}/repo - velocitas init - - - name: Check if vehicle model is generated - run: | - cd test/${{ matrix.language }}/repo - generatedModelPath=$(cat .velocitas.json | jq -r .variables.generatedModelPath) - if [ -d "$generatedModelPath" ]; - then - echo "$generatedModelPath directory exists." - else - echo "$generatedModelPath does not exist." - exit 1 + REF=main + if [ "${{ matrix.language }}" = "cpp" ]; then + REF=manifest-v3 fi + echo "ref=$REF" >> $GITHUB_OUTPUT - - name: Sync velocitas project - run: | - cd test/${{ matrix.language }}/repo - velocitas sync - - - name: Test for CLI Auto-Upgrade - run: | - cd test/${{ matrix.language }}/repo - pytest -s ../../integration/test_poststart.py - - - name: Reset .velocitas.json to latest template state - run: | - cd test/${{ matrix.language }}/repo - git checkout .velocitas.json + - name: Checkout template repository + uses: actions/checkout@v4 + with: + repository: eclipse-velocitas/vehicle-app-${{ matrix.language }}-template + path: test/${{ matrix.language }}/repo + ref: ${{ steps.get-repo-ref.outputs.ref }} - - name: Fix dubious ownership + - name: Add safe directory run: | - git config --global --add safe.directory $( pwd ) + git config --global --add safe.directory $(pwd)/test/${{ matrix.language }}/repo - - name: Identify changes in devenv repo - uses: dorny/paths-filter@v2 - id: changesInPath + - name: Update Velocitas JSON with correct version + uses: ./.github/actions/update-velocitas-json with: - filters: | - setupCommon: - - 'setup/src/common/**' - setupLang: - - 'setup/src/${{ matrix.language }}/**' - - - name: Check if cloned template repo has changes after sync - if: (steps.changesInPath.outputs.setupCommon == 'true' || steps.changesInPath.outputs.setupLang == 'true') - id: changes + project_path: test/${{ matrix.language }}/repo + template_path: test/${{ matrix.language }}/.velocitas.json + + - name: Run integration tests run: | - cd test/${{ matrix.language }}/repo - git diff - if [[ -z "$(git status --porcelain .)" ]]; - then - echo "No Changes" - echo "changed=0" >> $GITHUB_OUTPUT - else - echo "Changes" - echo "changed=1" >> $GITHUB_OUTPUT - fi - shell: bash + export THIS_REPO_PATH=$(pwd) + export VELOCITAS_TEST_LANGUAGE=${{ matrix.language }} + export VELOCITAS_TEST_ROOT=test/${{ matrix.language }}/repo - - name: Fail if there are changes in setup files detected but sync probably failed - if: | - (steps.changes.outputs.changed == 0 && steps.changesInPath.outputs.setupCommon == 'true') || - (steps.changes.outputs.changed == 0 && steps.changesInPath.outputs.setupLang == 'true') - run: exit 1 + python -m pip install -r ./${{ matrix.component }}/test/requirements.txt + python -m pip install -r ./${{ matrix.component }}/requirements.txt + pytest ./${{ matrix.component }}/test/integration diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c89a86fc..3411aa5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,11 +13,27 @@ repos: rev: 5.12.0 hooks: - id: isort + exclude: > + (?x)^( + .*_pb2.py| + .*_pb2.pyi| + .*_pb2_grpc.py| + .*conanfile.py| + .*templates.*.py + )$ - repo: https://github.com/psf/black rev: 23.1.0 hooks: - id: black + exclude: > + (?x)^( + .*_pb2.py| + .*_pb2.pyi| + .*_pb2_grpc.py| + .*conanfile.py| + .*templates.*.py + )$ - repo: https://github.com/PyCQA/flake8 rev: 6.0.0 @@ -27,7 +43,9 @@ repos: (?x)^( .*_pb2.py| .*_pb2.pyi| - .*_pb2_grpc.py + .*_pb2_grpc.py| + .*conanfile.py| + .*templates.*.py )$ additional_dependencies: - flake8-bugbear @@ -37,7 +55,7 @@ repos: rev: 1.7.4 hooks: - id: bandit - args: ["--skip=B101,B404,B603"] + args: ["--skip=B101,B404,B603,B607"] types_or: [python] exclude: test @@ -45,7 +63,7 @@ repos: rev: "v1.0.1" hooks: - id: mypy - args: [vehicle-model-lifecycle/src] + args: [--strict, --ignore-missing-imports, --exclude=conanfile.py, --exclude=.*templates.*.py, vehicle-model-lifecycle/src, grpc-interface-support/src] language: system pass_filenames: false diff --git a/README.md b/README.md index ae353a19..0c8c941e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ -# devenv-devcontainer-setup +# Velocitas Package for DevContainer Setup Repo for Devcontainer Configuration and vehicle model generation on container startup. + +## Components in this package + +* [Basic Setup](./setup/README.md) +* [gRPC Interface Support](./grpc-service-support/README.md) +* [Vehicle Signal Interface Support](./vehicle-model-lifecycle/README.md) +* [SDK Installer](./sdk-installer/README.md) diff --git a/grpc-interface-support/README.md b/grpc-interface-support/README.md new file mode 100644 index 00000000..54bad625 --- /dev/null +++ b/grpc-interface-support/README.md @@ -0,0 +1,5 @@ +# gRPC Interface Support + +Extends Velocitas applications and services with the ability to describe a dependency to a gRPC service via its interface definition in a proto file. + +The component provides CLI exec targets which will generate service client SDKs for all dependent services which will be made available for the application to use via a simple factory interface. diff --git a/grpc-interface-support/requirements.txt b/grpc-interface-support/requirements.txt new file mode 100644 index 00000000..ee428502 --- /dev/null +++ b/grpc-interface-support/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/eclipse-velocitas/velocitas-lib.git@v0.0.2 diff --git a/grpc-interface-support/src/__init__.py b/grpc-interface-support/src/__init__.py new file mode 100644 index 00000000..fdcc18ec --- /dev/null +++ b/grpc-interface-support/src/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/grpc-interface-support/src/conan_helper.py b/grpc-interface-support/src/conan_helper.py new file mode 100644 index 00000000..1463d3ac --- /dev/null +++ b/grpc-interface-support/src/conan_helper.py @@ -0,0 +1,220 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import glob +import os +import shutil +import subprocess +from typing import List, Optional, Tuple + +from util import to_camel_case +from util.templates import CopySpec, copy_templates +from velocitas_lib import get_package_path, get_workspace_dir + + +def get_required_sdk_version() -> Optional[str]: + """Return the required version of the core SDK. + + Returns: + Optional[str]: The required version or None in case SDK is not a dependency. + """ + sdk_version: Optional[str] = None + with open( + os.path.join(get_workspace_dir(), "conanfile.txt"), encoding="utf-8" + ) as conanfile: + for line in conanfile: + if line.startswith("vehicle-app-sdk/"): + sdk_version = line.split("/", maxsplit=1)[1].split("@")[0].strip() + + return sdk_version + + +def move_generated_sources( + generated_source_dir: str, output_dir: str, include_dir_rel: str, src_dir_rel: str +) -> Tuple[List[str], List[str]]: + """Move generated source code from the generation dir into + headers: / + sources: / + + Args: + generated_source_dir (str): The directory containing the generated sources. + output_dir (str): The root directory to move the generated files to. + include_dir_rel (str): Path relative to output_dir where to move the headers to. + src_dir_rel (str): Path relative to the output_dir where to move the sources to. + + Returns: + Tuple[List[str], List[str]]: A tuple containing + [0] = a list of all headers + [1] = a list of all sources + """ + + headers = glob.glob(os.path.join(generated_source_dir, "*.h")) + sources = glob.glob(os.path.join(generated_source_dir, "*.cc")) + + headers_relative = [] + for header in headers: + rel_path = os.path.relpath(header, generated_source_dir) + os.makedirs(os.path.join(output_dir, include_dir_rel), exist_ok=True) + shutil.move(header, os.path.join(output_dir, include_dir_rel, rel_path)) + headers_relative.append(os.path.join(include_dir_rel, rel_path)) + + sources_relative = [] + for source in sources: + rel_path = os.path.relpath(source, generated_source_dir) + os.makedirs(os.path.join(output_dir, src_dir_rel), exist_ok=True) + shutil.move(source, os.path.join(output_dir, src_dir_rel, rel_path)) + sources_relative.append(os.path.join(src_dir_rel, rel_path)) + + return headers_relative, sources_relative + + +def create_conan_project( + project_dir: str, interface_namespace: str, service_name: str +) -> None: + """Create a conan project in the given project directory. + + Args: + project_dir (str): The directory to create the project in. + interface_namespace (str): The namespace of the proto file. + service_name (str): The name of the service (from proto file). + """ + + include_dir = f"include/services/{service_name.lower()}" + src_dir = f"src/services/{service_name.lower()}" + + headers_relative, sources_relative = move_generated_sources( + project_dir, project_dir, include_dir, src_dir + ) + + files_to_copy = [ + CopySpec(source_path="CMakeLists.txt"), + CopySpec(source_path="conanfile.py"), + CopySpec( + "ServiceNameServiceClientFactory.h", + f"{include_dir}/{to_camel_case(service_name)}ServiceClientFactory.h", + ), + CopySpec( + "ServiceNameServiceClientFactory.cc", + f"{src_dir}/{to_camel_case(service_name)}ServiceClientFactory.cc", + ), + ] + + headers_relative.append( + f"{include_dir}/{to_camel_case(service_name)}ServiceClientFactory.h" + ) + sources_relative.append( + f"{src_dir}/{to_camel_case(service_name)}ServiceClientFactory.cc" + ) + + variables = { + "service_name": service_name, + "service_name_lower": service_name.lower(), + "service_name_camel_case": to_camel_case(service_name), + "headers": "\n\t".join(headers_relative), + "sources": "\n\t".join(sources_relative), + "package_id": interface_namespace.replace(".", "::"), + "core_sdk_version": str(get_required_sdk_version()), + } + + template_dir = os.path.join( + get_package_path(), "grpc-interface-support", "templates", "cpp" + ) + + copy_templates(template_dir, project_dir, files_to_copy, variables) + + +def export_conan_project(conan_project_path: str) -> None: + """Export a conan project to the local conan cache. + + Args: + conan_project_path (str): The path to directory containing the project. + """ + print("Exporting Conan project") + subprocess.check_call( + ["conan", "export", "."], + cwd=conan_project_path, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def _find_insertion_index( + lines: List[str], dependency_name: str +) -> Tuple[int, bool, bool]: + """Find an insertion index for the dependency in a conanfile.txt. + + Args: + lines (List[str]): The lines of the original conanfile.txt + dependency_name (str): The name of the dependency (without version) e.g. "grpc" + of the dependency to insert. + + Returns: + Tuple[int, bool, bool]: A tuple consisting of + [0] = Insert index. + [1] = Whether the insert index replaces the original line or not. + [2] = Whether the original file has a requires section or not. + """ + found_index: Optional[int] = None + replace: bool = False + in_requires_section = False + has_requires_section = False + for i in range(0, len(lines)): + stripped_line = lines[i].strip() + if stripped_line == "[requires]": + has_requires_section = True + in_requires_section = True + found_index = i + 1 + elif in_requires_section and stripped_line.startswith("["): + in_requires_section = False + + if in_requires_section: + if len(stripped_line) > 0: + if stripped_line.startswith(dependency_name): + found_index = i + replace = True + + if found_index is None: + found_index = len(lines) + + return (found_index, replace, has_requires_section) + + +def add_dependency_to_conanfile(dependency_name: str, dependency_version: str) -> None: + """Add the dependency name to the project's list of dependencies. + + Args: + dependency_name (str): The dependency to add e.g. grpc + dependency_version (str): The version of the dependency to add e.g. 1.50.1 + """ + conanfile_path = os.path.join(get_workspace_dir(), "conanfile.txt") + + lines = [] + with open(conanfile_path, encoding="utf-8", mode="r") as conanfile: + lines = conanfile.readlines() + + insert_index, replace, has_requires_section = _find_insertion_index( + lines, dependency_name + ) + + dependency_line = f"{dependency_name}/{dependency_version}\n" + if replace: + lines[insert_index] = dependency_line + else: + if not has_requires_section: + lines.insert(insert_index, "[requires]\n") + insert_index = insert_index + 1 + lines.insert(insert_index, dependency_line) + + with open(conanfile_path, encoding="utf-8", mode="w") as conanfile: + conanfile.writelines(lines) diff --git a/grpc-interface-support/src/cpp.py b/grpc-interface-support/src/cpp.py new file mode 100644 index 00000000..75f069c9 --- /dev/null +++ b/grpc-interface-support/src/cpp.py @@ -0,0 +1,167 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import conan_helper +from generator import GrpcInterfaceGenerator +from proto import ProtoFileHandle +from velocitas_lib import get_package_path + +CONAN_PROFILE_NAME = "host" + + +class CppGrpcInterfaceGenerator(GrpcInterfaceGenerator): # type: ignore + def __init__(self, verbose: bool): + self._verbose = verbose + + def __create_conan_profile(self) -> None: + subprocess.check_call( + ["conan", "profile", "new", CONAN_PROFILE_NAME, "--detect", "--force"], + stdout=subprocess.DEVNULL, + ) + + subprocess.check_call( + [ + "conan", + "profile", + "update", + "settings.compiler.libcxx=libstdc++11", + CONAN_PROFILE_NAME, + ], + stdout=subprocess.DEVNULL if not self._verbose else None, + ) + + subprocess.check_call( + [ + "conan", + "profile", + "update", + "settings.build_type=Release", + CONAN_PROFILE_NAME, + ], + stdout=subprocess.DEVNULL if not self._verbose else None, + ) + + def __install_protoc_via_conan(self, conan_build_dir: str) -> None: + env = os.environ.copy() + env["CONAN_REVISIONS_ENABLED"] = "1" + + template_dir = os.path.join( + get_package_path(), "grpc-interface-support", "templates", "cpp" + ) + + deps_to_extract = ["grpc", "c-ares", "googleapis", "grpc-proto"] + deps_patterns = [ + re.compile(r"^.*\"(" + dep + r"\/.*)\".*$") for dep in deps_to_extract + ] + deps_results = [] + with open( + os.path.join(template_dir, "conanfile.py"), encoding="utf-8" + ) as conanfile: + for line in conanfile: + for pattern in deps_patterns: + match = pattern.match(line) + if match is not None: + deps_results.append(match.group(1)) + + tmpdir = tempfile.mkdtemp() + tooling_conanfile_path = os.path.join(tmpdir, "conanfile.txt") + with open( + tooling_conanfile_path, mode="w", encoding="utf-8" + ) as tooling_conanfile: + tooling_conanfile.write("[requires]\n" + "\n".join(deps_results)) + + subprocess.check_call( + [ + "conan", + "install", + "-pr:h", + CONAN_PROFILE_NAME, + "--build", + "missing", + tooling_conanfile_path, + ], + cwd=conan_build_dir, + env=env, + stdout=subprocess.DEVNULL if not self._verbose else None, + ) + + def __extend_path_with_protoc_and_plugin(self, conan_install_dir: str) -> None: + with open(os.path.join(conan_install_dir, "conanbuildinfo.txt")) as buildinfo: + for line in buildinfo: + if line.startswith("PATH=["): + addition = line.replace('PATH=["', "").replace('"]', "").strip() + os.environ["PATH"] = os.environ["PATH"] + ":" + addition + + def install_tooling(self) -> None: + self.__create_conan_profile() + + conan_install_dir = tempfile.mkdtemp() + self.__install_protoc_via_conan(conan_install_dir) + self.__extend_path_with_protoc_and_plugin(conan_install_dir) + + def get_binary_path(self, binary_name: str) -> str: + path = shutil.which(binary_name) + if path is None: + raise KeyError(f"{binary_name!r} missing!") + return path + + def __invoke_code_generator( + self, proto_file_path: str, include_path: str, output_path: str + ) -> None: + print("Invoking gRPC code generator") + args = [ + self.get_binary_path("protoc"), + f"--plugin=protoc-gen-grpc={self.get_binary_path('grpc_cpp_plugin')}", + f"-I{include_path}", + f"--cpp_out={output_path}", + f"--grpc_out={output_path}", + proto_file_path, + ] + subprocess.check_call( + args, + cwd=include_path, + env=os.environ, + stdout=subprocess.DEVNULL if not self._verbose else None, + ) + + def generate_service_client_sdk( + self, output_path: str, proto_file_handle: ProtoFileHandle + ) -> None: + self.__invoke_code_generator( + proto_file_handle.file_path, + str(Path(proto_file_handle.file_path).parent), + output_path, + ) + + conan_helper.create_conan_project( + output_path, + proto_file_handle.get_package(), + proto_file_handle.get_service_name(), + ) + + conan_helper.export_conan_project(output_path) + + conan_helper.add_dependency_to_conanfile( + f"{proto_file_handle.get_service_name().lower()}-service-sdk", "generated" + ) + + def generate_service_server_sdk(self) -> None: + pass diff --git a/grpc-interface-support/src/generator.py b/grpc-interface-support/src/generator.py new file mode 100644 index 00000000..e647087f --- /dev/null +++ b/grpc-interface-support/src/generator.py @@ -0,0 +1,50 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod + +import proto + + +class GrpcInterfaceGenerator(ABC): + @abstractmethod + def install_tooling(self) -> None: + """Install required tooling for the generator.""" + pass + + @abstractmethod + def generate_service_client_sdk( + self, output_path: str, proto_file_handle: proto.ProtoFileHandle + ) -> None: + """Generate a service client for the given proto file. + + Args: + output_path (str): The path at which to output the client SDK. + proto_file_handle (proto.ProtoFileHandle): A proto file handle + which represents the service contract. + """ + pass + + @abstractmethod + def generate_service_server_sdk( + self, output_path: str, proto_file_handle: proto.ProtoFileHandle + ) -> None: + """Generate a service server for the given proto file. + + Args: + output_path (str): The path at which to output the server SDK. + proto_file_handle (proto.ProtoFileHandle): A proto file handle + which represents the service contract. + """ + pass diff --git a/grpc-interface-support/src/main.py b/grpc-interface-support/src/main.py new file mode 100644 index 00000000..9eca5f0c --- /dev/null +++ b/grpc-interface-support/src/main.py @@ -0,0 +1,132 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import shutil +from typing import Any, Dict + +import proto +from cpp import CppGrpcInterfaceGenerator +from generator import GrpcInterfaceGenerator +from python import PythonGrpcInterfaceGenerator +from util import create_truncated_string +from velocitas_lib import download_file, get_programming_language, get_project_cache_dir +from velocitas_lib.functional_interface import get_interfaces_for_type + +DEPENDENCY_TYPE_KEY = "grpc-interface" + + +def download_proto(config: Dict[str, Any]) -> proto.ProtoFileHandle: + """Download the proto file defined in the grpc-interface + config to the local project cache. + + Args: + config (Dict[str, Any]): The grpc-interface config. + + Returns: + proto.ProtoFileHandle: A handle to the proto file. + """ + service_if_spec_src = config["src"] + _, filename = os.path.split(service_if_spec_src) + + cached_proto_file_path = os.path.join(get_project_cache_dir(), "services", filename) + + download_file(service_if_spec_src, cached_proto_file_path) + + return proto.ProtoFileHandle(cached_proto_file_path) + + +def create_service_sdk_dir(proto_file_handle: proto.ProtoFileHandle) -> str: + """Create a directory for the service SDK. + + Args: + proto_file_handle (proto.ProtoFileHandle): + A handle to the proto file of the service. + + Returns: + str: The absolute path to the SDK directory. + """ + service_name = proto_file_handle.get_service_name() + service_sdk_path = os.path.join( + get_project_cache_dir(), "services", service_name.lower() + ) + if os.path.isdir(service_sdk_path): + shutil.rmtree(service_sdk_path) + os.makedirs(service_sdk_path, exist_ok=True) + + return service_sdk_path + + +def generate_single_service( + generator: GrpcInterfaceGenerator, if_config: Dict[str, Any] +) -> None: + """Generate an SDK for a single service. + + Args: + generator (GrpcInterfaceGenerator): + The generator to invoke for generating the SDK. + if_config (Dict[str, Any]): The grpc-interface config. + """ + print( + f"Generating service SDK for {create_truncated_string(if_config['src'], 40)!r}" + ) + proto_file_handle = download_proto(if_config) + service_sdk_dir = create_service_sdk_dir(proto_file_handle) + + if "requires" in if_config: + generator.generate_service_client_sdk(service_sdk_dir, proto_file_handle) + if "provides" in if_config: + pass + + +def main(verbose: bool) -> None: + """Generate service SDKs for all grpc-interfaces defined in the AppManifest.json. + + Args: + verbose (bool): Enable verbose logging. + """ + interfaces = get_interfaces_for_type(DEPENDENCY_TYPE_KEY) + + if len(interfaces) <= 0: + return + + LANGUAGE_GENERATORS = { + "cpp": CppGrpcInterfaceGenerator(verbose), + "python": PythonGrpcInterfaceGenerator(verbose), + } + + if get_programming_language() not in LANGUAGE_GENERATORS: + print( + "gRPC interface not yet supported for programming language " + f"{get_programming_language()!r}" + ) + return + + generator = LANGUAGE_GENERATORS[get_programming_language()] + + print("Installing tooling...") + generator.install_tooling() + + for grpc_service in interfaces: + if_config = grpc_service["config"] + print("Generating service SDK...") + generate_single_service(generator, if_config) + + +if __name__ == "__main__": + argument_parser = argparse.ArgumentParser("generate-sdk") + argument_parser.add_argument("-v", "--verbose", action="store_true") + args = argument_parser.parse_args() + main(args.verbose) diff --git a/grpc-interface-support/src/proto.py b/grpc-interface-support/src/proto.py new file mode 100644 index 00000000..344dc03d --- /dev/null +++ b/grpc-interface-support/src/proto.py @@ -0,0 +1,58 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +class ProtoFileHandle: + def __init__(self, file_path: str): + self.file_path = file_path + + def get_package(self) -> str: + """Return the package of the proto file. + + Raises: + RuntimeError: In case there is no package ID in the proto file. + + Returns: + str: The package of the proto file. + """ + package_id = None + with open(self.file_path, encoding="utf-8") as file: + for line in file: + if line.startswith("package"): + package_id = line.split(" ")[1][:-2] + + if package_id is None: + raise RuntimeError("No package ID found in proto file!") + + return package_id + + def get_service_name(self) -> str: + """Get the name of the service. + + Raises: + RuntimeError: In case there is no defined service in the proto file. + + Returns: + str: The name of the service. + """ + service_name = None + with open(self.file_path, encoding="utf-8") as file: + for line in file: + if line.startswith("service"): + service_name = line.split(" ")[1].strip() + + if service_name is None: + raise RuntimeError("No service name found in proto file!") + + return service_name diff --git a/grpc-interface-support/src/python.py b/grpc-interface-support/src/python.py new file mode 100644 index 00000000..de04185c --- /dev/null +++ b/grpc-interface-support/src/python.py @@ -0,0 +1,117 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import glob +import os +import shutil +import subprocess +from pathlib import Path + +import proto +from generator import GrpcInterfaceGenerator +from util import replace_in_file, to_camel_case +from util.templates import CopySpec, copy_templates +from velocitas_lib import get_package_path, get_workspace_dir + + +def get_required_sdk_version_python() -> str: + sdk_version: str = "0.11.0" + with open( + os.path.join(get_workspace_dir(), "app", "requirements.txt"), encoding="utf-8" + ) as requirements_file: + for line in requirements_file: + if line.startswith("vehicle-app-sdk"): + sdk_version = line.split("==")[1].strip() + + return sdk_version + + +class PythonGrpcInterfaceGenerator(GrpcInterfaceGenerator): # type: ignore + def __init__(self, verbose: bool): + self._verbose = verbose + + def install_tooling(self) -> None: + subprocess.check_call(["pip", "install", "grpcio-tools"]) + + def __invoke_code_generator( + self, proto_file_handle: proto.ProtoFileHandle, output_path: str + ) -> None: + subprocess.check_call( + [ + "python", + "-m", + "grpc_tools.protoc", + f"-I{Path(proto_file_handle.file_path).parent}", + f"--python_out={output_path}", + f"--pyi_out={output_path}", + f"--grpc_python_out={output_path}", + proto_file_handle.file_path, + ] + ) + + def __copy_code_and_templates(self, output_path: str, service_name: str) -> None: + module_name = f"{service_name.lower()}_service_sdk" + source_path = os.path.join(output_path, module_name) + os.makedirs(os.path.join(output_path, source_path), exist_ok=True) + + generated_sources = glob.glob(os.path.join(output_path, "*.py*")) + replace_in_file( + os.path.join(output_path, f"{service_name.lower()}_pb2_grpc.py"), + f"import {service_name.lower()}_pb2", + f"import {module_name}.{service_name.lower()}_pb2", + ) + + for file in generated_sources: + shutil.move(file, source_path) + + files_to_copy = [ + CopySpec( + source_path="ServiceNameServiceClientFactory.py", + target_path=os.path.join( + source_path, f"{service_name}ServiceClientFactory.py" + ), + ), + CopySpec(source_path="pyproject.toml"), + ] + + variables = { + "service_name": service_name, + "service_name_lower": service_name.lower(), + "service_name_camel_case": to_camel_case(service_name), + "core_sdk_version": get_required_sdk_version_python(), + } + + template_dir = os.path.join( + get_package_path(), + "grpc-interface-support", + "templates", + "python", + ) + + copy_templates(template_dir, output_path, files_to_copy, variables) + + def __install_module(self, module_path: str) -> None: + subprocess.check_call(["pip", "install", module_path]) + + def generate_service_client_sdk( + self, output_path: str, proto_file_handle: proto.ProtoFileHandle + ) -> None: + self.__invoke_code_generator(proto_file_handle, output_path) + self.__copy_code_and_templates( + output_path, proto_file_handle.get_service_name() + ) + self.__install_module(output_path) + + def generate_service_server_sdk(self) -> None: + pass diff --git a/grpc-interface-support/src/util/__init__.py b/grpc-interface-support/src/util/__init__.py new file mode 100644 index 00000000..87c0fbdf --- /dev/null +++ b/grpc-interface-support/src/util/__init__.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + + +def to_camel_case(snake_str: str) -> str: + """Return a camel case version of a snake case string. + + Args: + snake_str (str): A snake case string. + + Returns: + str: A camel case version of a snake case string. + """ + return "".join(x.capitalize() for x in snake_str.lower().split("-")) + + +def create_truncated_string(input: str, length: int) -> str: + """Create a truncated version of input if it is longer than length. + Will keep the rightmost characters and cut of the front if it is + longer than allowed. + + Args: + input (str): The input string. + length (int): The allowed overall length. + + Returns: + str: A truncated string which has len() of length. + """ + if len(input) < length: + return input + + return f"...{input[-length+3:]}" + + +def replace_in_file(file_path: str, text: str, replacement: str) -> None: + """Replace all occurrences of text in a file with a replacement. + + Args: + file_path (str): The path to the file. + text (str): The text to find. + replacement (str): The replacement for text. + """ + buffer = [] + for line in open(file_path, encoding="utf-8"): + buffer.append(line.replace(text, replacement)) + + with open(file_path, mode="w") as file: + for line in buffer: + file.write(line) diff --git a/grpc-interface-support/src/util/templates.py b/grpc-interface-support/src/util/templates.py new file mode 100644 index 00000000..b6a02951 --- /dev/null +++ b/grpc-interface-support/src/util/templates.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from typing import Dict, List, Optional + + +class CopySpec: + """Copy specification of a single file or directory.""" + + def __init__(self, source_path: str, target_path: Optional[str] = None): + self.source_path = source_path + self.target_path = target_path + + + def get_target(self) -> str: + """Get the target path of the copy spec. + + Returns: + str: If a target_path is given explicitly, it will be returned. + Otherwise the source_path will be returned. + """ + if self.target_path is not None: + return self.target_path + return self.source_path + + +def copy_templates( + template_dir: str, + target_dir: str, + template_file_mapping: List[CopySpec], + variables: Dict[str, str], +) -> None: + """Copy templates from the template dir to the target dir. + + Args: + template_dir (str): Path to the directory containing the template files. + target_dir (str): Path to the target directory. + template_file_mapping (Dict[str, str]): A mapping of source path to target path. + variables (Dict[str, str]): Name to value mapping which will be replaced when parsing the template files. + """ + for file_to_copy in template_file_mapping: + with open( + os.path.join( + template_dir, + file_to_copy.source_path, + ), + encoding="utf-8", + ) as file_in: + target_file_path = os.path.join(target_dir, file_to_copy.get_target()) + + os.makedirs(os.path.split(target_file_path)[0], exist_ok=True) + with open(target_file_path, encoding="utf-8", mode="w") as file_out: + lines = file_in.readlines() + for line in lines: + for key, value in variables.items(): + line = line.replace("${{ " + key + " }}", value) + file_out.write(line) diff --git a/grpc-interface-support/templates/cpp/CMakeLists.txt b/grpc-interface-support/templates/cpp/CMakeLists.txt new file mode 100644 index 00000000..45424851 --- /dev/null +++ b/grpc-interface-support/templates/cpp/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.15) + +set(PROJECT_NAME ${{ service_name_lower }}-service-sdk) + +project(${PROJECT_NAME} CXX) + +add_library(${PROJECT_NAME} + ${{ sources }} +) + +target_include_directories(${PROJECT_NAME} + PUBLIC include +) + +include_directories( + ${CMAKE_INCLUDE_PATH} + include/services/${{ service_name_lower }} +) + +set(HEADERS + ${{ headers }} +) + +set_target_properties(${PROJECT_NAME} PROPERTIES + PUBLIC_HEADER "${HEADERS}" +) + +install(TARGETS ${PROJECT_NAME} + PUBLIC_HEADER DESTINATION include/services/${{ service_name_lower }} +) diff --git a/grpc-interface-support/templates/cpp/ServiceNameServiceClientFactory.cc b/grpc-interface-support/templates/cpp/ServiceNameServiceClientFactory.cc new file mode 100644 index 00000000..c625abc0 --- /dev/null +++ b/grpc-interface-support/templates/cpp/ServiceNameServiceClientFactory.cc @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "services/${{ service_name_lower }}/${{ service_name_camel_case }}ServiceClientFactory.h" + +#include "sdk/middleware/Middleware.h" + +#include +#include +#include + +namespace velocitas { + +std::shared_ptr<${{ package_id }}::${{ service_name }}::Stub> +${{ service_name_camel_case }}ServiceClientFactory::create(Middleware& middleware) { + auto channel = grpc::CreateChannel(middleware.getServiceLocation("${{ service_name }}"), grpc::InsecureChannelCredentials()); + auto stub = std::make_shared<${{ package_id }}::${{ service_name }}::Stub>(channel); + return stub; +} + +} // namespace velocitas diff --git a/grpc-interface-support/templates/cpp/ServiceNameServiceClientFactory.h b/grpc-interface-support/templates/cpp/ServiceNameServiceClientFactory.h new file mode 100644 index 00000000..d10283d1 --- /dev/null +++ b/grpc-interface-support/templates/cpp/ServiceNameServiceClientFactory.h @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef VELOCITAS_SERVICE_${{ service_name }}_CLIENT_FACTORY_H +#define VELOCITAS_SERVICE_${{ service_name }}_CLIENT_FACTORY_H + +#include "services/${{ service_name_lower }}/${{ service_name_lower }}.grpc.pb.h" + +#include + +namespace velocitas { + +class Middleware; + +/** + * @brief Factory which facilitates creation of ${{ service_name_camel_case }}. + * + */ +class ${{ service_name_camel_case }}ServiceClientFactory { +public: + /** + * @brief Create a new ${{ service_name_camel_case }} client. + * + * @param middleware The middleware used by the Velocitas application. + * + * @return A new ${{ service_name_camel_case }} instance. + */ + static std::shared_ptr<${{ package_id }}::${{ service_name }}::Stub> create(Middleware& middleware); + + ${{ service_name_camel_case }}ServiceClientFactory() = delete; +}; + +} // namespace velocitas + +#endif // VELOCITAS_SERVICE_${{ service_name }}_CLIENT_FACTORY_H diff --git a/grpc-interface-support/templates/cpp/conanfile.py b/grpc-interface-support/templates/cpp/conanfile.py new file mode 100644 index 00000000..6fd2dd30 --- /dev/null +++ b/grpc-interface-support/templates/cpp/conanfile.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout + + +class ${{ service_name_camel_case }}ServiceConan(ConanFile): + name = "${{ service_name_lower }}-service-sdk" + version = "generated" + + # Optional metadata + license = "Apache-2.0" + author = "Eclipse Velocitas Contributors" + url = "https://github.com/eclipse-velocitas/devenv-devcontainer-setup" + description = "Auto-generated SDK for ${{ service_name_camel_case }}" + topics = ("${{ service_name_camel_case }}", "gRPC", "RPC") + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + # Sources are located in the same place as this recipe, copy them to the recipe + exports_sources = "CMakeLists.txt", "src/*", "include/*" + + requires = [ + ("c-ares/1.19.1@#420a0b77e370f4b96bee88ef91837ccc"), + ("googleapis/cci.20221108@#e4bebdfa02f3b6f93bae1d5001b8d439"), + ("grpc/1.50.1@#df352027120f88bccf24cbc40a2297ce"), + ("grpc-proto/cci.20220627@#3ad14e3ffdae516b4da2407d5f23c71d"), + ("openssl/1.1.1u@#de76bbea24d8b46f8def8daa18b31fd9"), + ("protobuf/3.21.9@#515ceb0a1653cf84363d9968b812d6be"), + ("vehicle-app-sdk/${{ core_sdk_version }}") + ] + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + + def layout(self): + cmake_layout(self) + + def generate(self): + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = ["${{ service_name_lower }}-service-sdk"] diff --git a/grpc-interface-support/templates/python/ServiceNameServiceClientFactory.py b/grpc-interface-support/templates/python/ServiceNameServiceClientFactory.py new file mode 100644 index 00000000..36a67fa7 --- /dev/null +++ b/grpc-interface-support/templates/python/ServiceNameServiceClientFactory.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import grpc +from ${{ service_name_lower }}_service_sdk.${{ service_name_lower }}_pb2_grpc import ( + ${{ service_name }}Stub, +) +from velocitas_sdk.base import Middleware + + +class ${{ service_name_camel_case }}ServiceClientFactory: + @staticmethod + def create(middleware: Middleware) -> ${{ service_name }}Stub: + address = middleware.service_locator.get_service_location("${{ service_name }}") + channel = grpc.aio.insecure_channel(address) + + return ${{ service_name }}Stub(channel) diff --git a/grpc-interface-support/templates/python/pyproject.toml b/grpc-interface-support/templates/python/pyproject.toml new file mode 100644 index 00000000..25122536 --- /dev/null +++ b/grpc-interface-support/templates/python/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "${{ service_name_lower }}-service-sdk" +version = "1.0.0" +dependencies = [ + "grpcio >= 1.57.0" +] + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" diff --git a/grpc-interface-support/test/integration/conftest.py b/grpc-interface-support/test/integration/conftest.py new file mode 100644 index 00000000..ddfeb129 --- /dev/null +++ b/grpc-interface-support/test/integration/conftest.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import subprocess + + +def pytest_sessionstart(session): + """ + Called after the Session object has been created and + before performing collection and entering the run test loop. + """ + + os.chdir(os.environ["VELOCITAS_TEST_ROOT"]) + shutil.copy("../../common/AppManifest.json", "./app/AppManifest.json") + + if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp": + shutil.copy("../app_seat_service_client/SampleApp.h", "./app/src/SampleApp.h") + shutil.copy( + "../app_seat_service_client/SampleApp.cpp", "./app/src/SampleApp.cpp" + ) + + # FIXME: The C++ base image does not install conan globally + # but just for the vscode user, hence we have to download + # conan manually here. Can be removed once conan is installed + # globally. + subprocess.check_call(["python", "-m", "pip", "install", "conan==1.60.2"]) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) diff --git a/grpc-interface-support/test/integration/test_integration_cpp.py b/grpc-interface-support/test/integration/test_integration_cpp.py new file mode 100644 index 00000000..148a0706 --- /dev/null +++ b/grpc-interface-support/test/integration/test_integration_cpp.py @@ -0,0 +1,64 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +from typing import List + +import pytest + +if not os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp": + pytest.skip("skipping C++ only tests", allow_module_level=True) + + +def get_subdirs(path: str) -> List[str]: + return [f.path for f in os.scandir(path) if f.is_dir()] + + +def get_project_cache_dir() -> str: + project_caches = os.path.join(os.path.expanduser("~"), ".velocitas", "projects") + return get_subdirs(project_caches)[0] + + +def test_grpc_package_is_generated(): + service_path = os.path.join(get_project_cache_dir(), "services", "seats") + + assert os.path.isdir(service_path) + assert os.path.isfile(os.path.join(service_path, "CMakeLists.txt")) + assert os.path.isfile(os.path.join(service_path, "conanfile.py")) + assert os.path.isdir(os.path.join(service_path, "include")) + assert os.path.isdir(os.path.join(service_path, "src")) + + +def test_project_depends_on_grpc_package(): + # Use a number rather than a bool to ensure + # the generator has added the dependency only + # once. + dependency_count = 0 + in_requires_section = False + for line in open("./conanfile.txt"): + if line.strip() == "[requires]": + in_requires_section = True + elif line.startswith("["): + in_requires_section = False + + if in_requires_section and line.strip() == "seats-service-sdk/generated": + dependency_count = dependency_count + 1 + + assert dependency_count == 1 + + +def test_project_is_buildable(): + assert subprocess.check_call(["./install_dependencies.sh"]) == 0 + assert subprocess.check_call(["./build.sh"]) == 0 diff --git a/grpc-interface-support/test/integration/test_integration_python.py b/grpc-interface-support/test/integration/test_integration_python.py new file mode 100644 index 00000000..0dbe6410 --- /dev/null +++ b/grpc-interface-support/test/integration/test_integration_python.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +from typing import List, Optional + +import pytest + +if not os.environ["VELOCITAS_TEST_LANGUAGE"] == "python": + pytest.skip("skipping Python only tests", allow_module_level=True) + + +def get_subdirs(path: str) -> List[str]: + return [f.path for f in os.scandir(path) if f.is_dir()] + + +def get_project_cache_dir() -> str: + project_caches = os.path.join(os.path.expanduser("~"), ".velocitas", "projects") + return get_subdirs(project_caches)[0] + + +def test_python_package_is_generated(): + service_path = os.path.join(get_project_cache_dir(), "services", "seats") + assert os.path.isdir(service_path) + assert os.path.isfile(os.path.join(service_path, "pyproject.toml")) + + source_path = os.path.join(service_path, "seats_service_sdk") + assert os.path.isdir(source_path) + assert os.path.isfile(os.path.join(source_path, "seats_pb2_grpc.py")) + assert os.path.isfile(os.path.join(source_path, "seats_pb2.py")) + assert os.path.isfile(os.path.join(source_path, "seats_pb2.pyi")) + + +def test_pip_package_is_usable(): + from seats_service_sdk.SeatsServiceClientFactory import SeatsServiceClientFactory + from velocitas_sdk.base import Middleware, ServiceLocator + + class TestServiceLocator(ServiceLocator): + def get_service_location(self, service_name: str) -> str: + return f"{service_name}@anyserver:anyport" + + def get_metadata(self, service_name: Optional[str] = None): + pass + + class TestMiddleware(Middleware): + def __init__(self) -> None: + self.service_locator = TestServiceLocator() + + async def start(self): + pass + + async def wait_until_ready(self): + pass + + async def stop(self): + pass + + middleware = TestMiddleware() + client = SeatsServiceClientFactory.create(middleware) + + assert client is not None diff --git a/grpc-interface-support/test/requirements.txt b/grpc-interface-support/test/requirements.txt new file mode 100644 index 00000000..cc6a8770 --- /dev/null +++ b/grpc-interface-support/test/requirements.txt @@ -0,0 +1,8 @@ +pytest +pytest-ordering +pytest-asyncio +pytest-cov +pyfakefs +types-mock +coverage2clover +coveragepy-lcov diff --git a/grpc-interface-support/test/unit/test_conan_helper.py b/grpc-interface-support/test/unit/test_conan_helper.py new file mode 100644 index 00000000..2bb66ce4 --- /dev/null +++ b/grpc-interface-support/test/unit/test_conan_helper.py @@ -0,0 +1,118 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +import pytest +from pyfakefs.fake_filesystem import FakeFilesystem +from velocitas_lib import get_workspace_dir + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "src")) +from conan_helper import add_dependency_to_conanfile # noqa + + +@pytest.fixture +def env(): + os.environ["VELOCITAS_WORKSPACE_DIR"] = "/home/workspace" + + +def test_add_dependency_to_conanfile__only_requires_section(fs: FakeFilesystem, env): + contents = """ +[requires] +""" + + conanfile_path = os.path.join(get_workspace_dir(), "conanfile.txt") + fs.create_file(conanfile_path, contents=contents) + add_dependency_to_conanfile("mydep", "myver") + + expected = """ +[requires] +mydep/myver +""" + + assert expected == open(conanfile_path, encoding="utf-8").read() + + +def test_add_dependency_to_conanfile__multiple_sections(fs: FakeFilesystem, env): + contents = """ +[requires] + +[foo] + +[bar] +""" + + conanfile_path = os.path.join(get_workspace_dir(), "conanfile.txt") + fs.create_file(conanfile_path, contents=contents) + add_dependency_to_conanfile("mydep", "myver") + + expected = """ +[requires] +mydep/myver + +[foo] + +[bar] +""" + + assert expected == open(conanfile_path, encoding="utf-8").read() + + +def test_add_dependency_to_conanfile__no_requires_section(fs: FakeFilesystem, env): + contents = """ +[foo] + +[bar] +""" + + conanfile_path = os.path.join(get_workspace_dir(), "conanfile.txt") + fs.create_file(conanfile_path, contents=contents) + add_dependency_to_conanfile("mydep", "myver") + + expected = """ +[foo] + +[bar] +[requires] +mydep/myver +""" + + assert expected == open(conanfile_path, encoding="utf-8").read() + + +def test_add_dependency_to_conanfile__pre_existing_dep_(fs: FakeFilesystem, env): + contents = """ +[foo] + +[requires] +mydep/myver2 + +[bar] +""" + + conanfile_path = os.path.join(get_workspace_dir(), "conanfile.txt") + fs.create_file(conanfile_path, contents=contents) + add_dependency_to_conanfile("mydep", "myver") + + expected = """ +[foo] + +[requires] +mydep/myver + +[bar] +""" + + assert expected == open(conanfile_path, encoding="utf-8").read() diff --git a/manifest.json b/manifest.json index e9d6d728..31deeec6 100644 --- a/manifest.json +++ b/manifest.json @@ -56,17 +56,17 @@ { "id": "install-deps", "executable": "python", - "args": ["./vehicle-model-lifecycle/src/install_deps.py"] + "args": [ "./vehicle-model-lifecycle/src/install_deps.py" ] }, { "id": "download-vspec", "executable": "python", - "args": ["./vehicle-model-lifecycle/src/download_vspec.py"] + "args": [ "./vehicle-model-lifecycle/src/download_vspec.py" ] }, { "id": "generate-model", "executable": "python", - "args": ["./vehicle-model-lifecycle/src/generate_model.py"] + "args": [ "./vehicle-model-lifecycle/src/generate_model.py" ] } ], "variables": [ @@ -95,6 +95,55 @@ "description": "Git ref of the model generator repo. Can be a tag, branch or SHA" } ] + }, + { + "id": "sdk-installer", + "type": "setup", + "onPostInit": [ + { + "ref": "install-deps" + }, + { + "ref": "run" + } + ], + "programs": [ + { + "id": "install-deps", + "executable": "python", + "args": [ "-m", "pip", "install", "-r", "./sdk-installer/requirements.txt" ] + }, + { + "id": "run", + "executable": "python", + "args": [ "./sdk-installer/src/run.py" ] + } + ] + }, + { + "id": "grpc-interface-support", + "type": "setup", + "onPostInit": [ + { + "ref": "install-deps" + }, + { + "ref": "generate-sdk" + } + ], + "programs": [ + { + "id": "install-deps", + "executable": "python", + "args": [ "-m", "pip", "install", "-r", "./grpc-interface-support/requirements.txt" ] + }, + { + "id": "generate-sdk", + "description": "Generates service client SDKs to be used in business logic.", + "executable": "python", + "args": [ "./grpc-interface-support/src/main.py" ] + } + ] } ] } diff --git a/sdk-installer/README.md b/sdk-installer/README.md new file mode 100644 index 00000000..44dbdb45 --- /dev/null +++ b/sdk-installer/README.md @@ -0,0 +1,7 @@ +# SDK Installer + +This component installs the Velocitas core SDK if it is referenced by the Vehicle Applications requirements file. + +## About + +At the moment, both core SDKs are not available through central package repositories (PyPI, Conan Center). Therefore, to use the core SDKs, they need to be made available to the package managers through other means. This is where this component comes in. It finds the required version from either `app/requirements.txt` or `conanfile.txt`, clones the respective SDK's git repository and exports the package to the local package registry. diff --git a/sdk-installer/requirements.txt b/sdk-installer/requirements.txt new file mode 100644 index 00000000..98e55f8f --- /dev/null +++ b/sdk-installer/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/eclipse-velocitas/velocitas-lib.git@v0.0.2 +cloudevents # FIXME: this dep is missing in SDK! diff --git a/sdk-installer/src/run.py b/sdk-installer/src/run.py new file mode 100644 index 00000000..5c3f5662 --- /dev/null +++ b/sdk-installer/src/run.py @@ -0,0 +1,148 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import re +import shutil +import subprocess +from typing import Optional + +from velocitas_lib import ( + get_programming_language, + get_project_cache_dir, + get_workspace_dir, +) + +SUPPORTED_LANGUAGES = ["cpp", "python"] + + +def get_required_sdk_version_cpp() -> str: + """Return the required version of the C++ SDK. + + Returns: + str: The required version. + """ + sdk_version: str = "0.4.0" + with open( + os.path.join(get_workspace_dir(), "conanfile.txt"), encoding="utf-8" + ) as conanfile: + for line in conanfile: + if line.startswith("vehicle-app-sdk/"): + sdk_version = line.split("/", maxsplit=1)[1].split("@")[0].strip() + + return sdk_version + + +def get_required_sdk_version_python() -> str: + """Return the required version of the Python SDK. + + Returns: + str: The required version. + """ + sdk_version: str = "0.12.0" + requirements_path = os.path.join( + get_workspace_dir(), "app", "requirements-velocitas.txt" + ) + if os.path.exists(requirements_path): + with open(requirements_path, encoding="utf-8") as requirements_file: + for line in requirements_file: + if line.startswith("velocitas-sdk"): + sdk_version = line.split("==")[1].strip() + + return sdk_version + + +def get_tag_or_branch_name(tag_or_branch_name: str) -> str: + """Return the tag or branch name of a git ref. + + Args: + tag_or_branch_name (str): A git ref. + + Returns: + str: The version tag (prepended with a 'v' prefix) + or the (unchanged) tag/branch name. + """ + version_tag_pattern = re.compile(r"^[0-9]+(\.[0-9]+){0,2}$") + if version_tag_pattern.match(tag_or_branch_name): + return f"v{tag_or_branch_name}" + return tag_or_branch_name + + +def main(verbose: bool): + """Installs the SDKs of the supported languages. + + Args: + verbose (bool): Enable verbose logging. + """ + lang = get_programming_language() + if lang not in SUPPORTED_LANGUAGES: + print("gRPC interface not yet supported for programming language " f"{lang!r}") + return + + required_sdk_version: Optional[str] = None + sdk_install_path = os.path.join(get_project_cache_dir(), f"vehicle-app-{lang}-sdk") + git_url = f"https://github.com/eclipse-velocitas/vehicle-app-{lang}-sdk.git" + + if lang == "cpp": + required_sdk_version = get_required_sdk_version_cpp() + elif lang == "python": + required_sdk_version = get_required_sdk_version_python() + + if required_sdk_version is None: + print("No SDK dependency detected -> Skipping installation.") + return + + print(f"Installing version {required_sdk_version!r}...") + + if os.path.exists(sdk_install_path): + shutil.rmtree(sdk_install_path) + + subprocess.check_call( + [ + "git", + "clone", + "--depth", + "1", + "-b", + get_tag_or_branch_name(required_sdk_version), + git_url, + ], + cwd=get_project_cache_dir(), + ) + + subprocess.check_call( + ["git", "config", "--global", "--add", "safe.directory", sdk_install_path], + stdout=subprocess.DEVNULL if not verbose else None, + ) + + if lang == "cpp": + subprocess.check_call( + ["conan", "export", "."], + stdout=subprocess.DEVNULL if not verbose else None, + cwd=sdk_install_path, + ) + elif lang == "python": + subprocess.check_call( + ["python", "-m", "pip", "install", "."], + stdout=subprocess.DEVNULL if not verbose else None, + cwd=sdk_install_path, + ) + + +if __name__ == "__main__": + argument_parser = argparse.ArgumentParser("generate-sdk") + argument_parser.add_argument("-v", "--verbose", action="store_true") + args = argument_parser.parse_args() + main(args.verbose) diff --git a/sdk-installer/test/integration/conftest.py b/sdk-installer/test/integration/conftest.py new file mode 100644 index 00000000..3ad72051 --- /dev/null +++ b/sdk-installer/test/integration/conftest.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import subprocess + + +def pytest_sessionstart(session): + """ + Called after the Session object has been created and + before performing collection and entering the run test loop. + """ + + os.chdir(os.environ["VELOCITAS_TEST_ROOT"]) + shutil.copy("../../common/AppManifest.json", "./app/AppManifest.json") + + # FIXME: The C++ base image does not install conan globally + # but just for the vscode user, hence we have to download + # conan manually here. Can be removed once conan is installed + # globally. + if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp": + subprocess.check_call(["python", "-m", "pip", "install", "conan==1.60.2"]) diff --git a/sdk-installer/test/integration/test_integration_cpp.py b/sdk-installer/test/integration/test_integration_cpp.py new file mode 100644 index 00000000..96b9d457 --- /dev/null +++ b/sdk-installer/test/integration/test_integration_cpp.py @@ -0,0 +1,52 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess + +import pytest + +if not os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp": + pytest.skip("skipping C++ only tests", allow_module_level=True) + + +def is_package_installed(package_name: str) -> bool: + output = subprocess.check_output( + ["conan", "search", package_name], encoding="utf-8" + ) + return output.find("Existing package recipes:") != -1 + + +def test_no_sdk_reference_found__latest_installed(): + conanfile_contents = """ +[requires] + + """ + with open("./conanfile.txt", mode="w") as conanfile: + conanfile.write(conanfile_contents) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) + assert is_package_installed("vehicle-app-sdk") + + +def test_sdk_reference_found__sdk_installed(): + conanfile_contents = """ +[requires] +vehicle-app-sdk/0.3.3 + """ + with open("./conanfile.txt", mode="w") as conanfile: + conanfile.write(conanfile_contents) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) + assert is_package_installed("vehicle-app-sdk") diff --git a/sdk-installer/test/integration/test_integration_python.py b/sdk-installer/test/integration/test_integration_python.py new file mode 100644 index 00000000..ab0b281f --- /dev/null +++ b/sdk-installer/test/integration/test_integration_python.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess + +import pytest + +if not os.environ["VELOCITAS_TEST_LANGUAGE"] == "python": + pytest.skip("skipping Python only tests", allow_module_level=True) + + +def is_package_installed(package_name: str) -> bool: + output = subprocess.check_output(["pip", "show", package_name], encoding="utf-8") + return output.find(f"Name: {package_name}") != -1 + + +def can_import_and_use_vehicleapp() -> bool: + try: + from velocitas_sdk.vehicle_app import VehicleApp + + class TestVehicleApp(VehicleApp): + def __init__(self): + pass + + app = TestVehicleApp() + return app is not None + except Exception as e: + print("Exception:" + str(e)) + return False + + +def test_no_sdk_reference_found__latest_version_installed(): + requirements_contents = """ + + """ + with open("./app/requirements-velocitas.txt", mode="w") as conanfile: + conanfile.write(requirements_contents) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) + assert is_package_installed("velocitas-sdk") + assert can_import_and_use_vehicleapp() + + +def test_sdk_reference_found__sdk_installed(): + requirements_contents = """ +velocitas-sdk==0.12.0 + """ + with open("./app/requirements-velocitas.txt", mode="w") as conanfile: + conanfile.write(requirements_contents) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) + assert is_package_installed("velocitas-sdk") + assert can_import_and_use_vehicleapp() diff --git a/sdk-installer/test/requirements.txt b/sdk-installer/test/requirements.txt new file mode 100644 index 00000000..996f2a29 --- /dev/null +++ b/sdk-installer/test/requirements.txt @@ -0,0 +1,7 @@ +pytest +pytest-ordering +pytest-asyncio +pytest-cov +types-mock +coverage2clover +coveragepy-lcov diff --git a/sdk-installer/test/unit/test_run.py b/sdk-installer/test/unit/test_run.py new file mode 100644 index 00000000..c0bbfdd8 --- /dev/null +++ b/sdk-installer/test/unit/test_run.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "src")) +from run import get_tag_or_branch_name # noqa + + +def test_get_tag_or_branch_name__semver_input__returns_valid_tag(): + assert "v0.1.0" == get_tag_or_branch_name("0.1.0") + assert "v2.0" == get_tag_or_branch_name("2.0") + assert "v3" == get_tag_or_branch_name("3") + + +def test_get_tag_or_branch_name__branch_name_input__returns_branch_name(): + assert "foo" == get_tag_or_branch_name("foo") + assert "1bar" == get_tag_or_branch_name("1bar") + assert "baz2" == get_tag_or_branch_name("baz2") diff --git a/setup/README.md b/setup/README.md new file mode 100644 index 00000000..4e2c938f --- /dev/null +++ b/setup/README.md @@ -0,0 +1,8 @@ +# Basic Setup + +This component defines the basic setup of the Velocitas dev-containers. +It is separated into: +* `src/common`: Contains setup (scripts) common to all container types (type = e.g. programming language) +* `src//common`: Setup common to language related SDK and template repositories +* `src//sdk`: Setup for language related SDK repository +* `src//app`: Setup for language related app template repository diff --git a/setup/requirements.txt b/setup/requirements.txt new file mode 100644 index 00000000..ee428502 --- /dev/null +++ b/setup/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/eclipse-velocitas/velocitas-lib.git@v0.0.2 diff --git a/setup/test/integration/conftest.py b/setup/test/integration/conftest.py new file mode 100644 index 00000000..5a55a278 --- /dev/null +++ b/setup/test/integration/conftest.py @@ -0,0 +1,18 @@ +import os +import subprocess + + +def pytest_sessionstart(session): + """ + Called after the Session object has been created and + before performing collection and entering the run test loop. + """ + + os.chdir(os.environ["VELOCITAS_TEST_ROOT"]) + + # FIXME: The C++ base image does not install conan globally + # but just for the vscode user, hence we have to download + # conan manually here. Can be removed once conan is installed + # globally. + if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp": + subprocess.check_call(["python", "-m", "pip", "install", "conan==1.60.2"]) diff --git a/test/integration/test_poststart.py b/setup/test/integration/test_integration.py similarity index 70% rename from test/integration/test_poststart.py rename to setup/test/integration/test_integration.py index e167b90c..cf527149 100644 --- a/test/integration/test_poststart.py +++ b/setup/test/integration/test_integration.py @@ -15,6 +15,7 @@ import json import os import platform +import subprocess from re import Pattern, compile, search from subprocess import PIPE, Popen, check_output @@ -81,6 +82,41 @@ def test_post_start_auto_upgrade_cli(): post_create_script_process.wait() - velocitas_json = open(os.path.join(os.getcwd(), ".velocitas.json")) + velocitas_json = open(".velocitas.json") data = json.load(velocitas_json) assert data["cliVersion"] == get_cli_version() + + +def test_files_synced(): + repo_path = os.environ["THIS_REPO_PATH"] + print(repo_path) + # check if there are any changes in the files to sync + changed_files = ( + subprocess.check_output( + ["git", "diff", "origin/main", "--name-only"], + cwd=repo_path, + ) + .decode() + .split("\n") + ) + language = os.environ["VELOCITAS_TEST_LANGUAGE"] + + changes_in_common = False + changes_in_lang = False + for changed_file in changed_files: + changes_in_common = changes_in_common or ( + changed_file.find("setup/src/common") != -1 + ) + changes_in_lang = changes_in_lang or ( + changed_file.find(f"setup/src/{language}") != -1 + ) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) + subprocess.check_call(["velocitas", "sync"]) + + git_status_output = subprocess.check_output( + ["git", "status", "--porcelain", "."] + ).decode() + + if changes_in_common or changes_in_lang: + assert git_status_output != "" diff --git a/setup/test/requirements.txt b/setup/test/requirements.txt new file mode 100644 index 00000000..fb1b0cc9 --- /dev/null +++ b/setup/test/requirements.txt @@ -0,0 +1,7 @@ +pytest==7.2.1 +pytest-ordering==0.6 +pytest-asyncio==0.20.3 +pytest-cov==4.0.0 +types-mock +coverage2clover==3.3.0 +coveragepy-lcov==0.1.2 diff --git a/setup/test/unit/test_dummy.py b/setup/test/unit/test_dummy.py new file mode 100644 index 00000000..0e2b8ca4 --- /dev/null +++ b/setup/test/unit/test_dummy.py @@ -0,0 +1,19 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# TODO: Fill me + + +def test_dummy(): + assert True diff --git a/test/common/AppManifest.json b/test/common/AppManifest.json new file mode 100644 index 00000000..7b7ee6df --- /dev/null +++ b/test/common/AppManifest.json @@ -0,0 +1,13 @@ +{ + "manifestVersion": "v3", + "name": "SampleApp", + "interfaces": [ + { + "type": "grpc-interface", + "config": { + "src": "https://raw.githubusercontent.com/eclipse/kuksa.val.services/main/seat_service/proto/sdv/edge/comfort/seats/v1/seats.proto", + "requires": { } + } + } + ] +} diff --git a/test/cpp/app_seat_service_client/SampleApp.cpp b/test/cpp/app_seat_service_client/SampleApp.cpp new file mode 100644 index 00000000..e72cdd4d --- /dev/null +++ b/test/cpp/app_seat_service_client/SampleApp.cpp @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2023 Robert Bosch GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "SampleApp.h" +#include "sdk/IPubSubClient.h" +#include "sdk/Logger.h" +#include "sdk/vdb/IVehicleDataBrokerClient.h" + +#include "sdk/middleware/Middleware.h" +#include "services/seats/SeatsServiceClientFactory.h" + +#include + +namespace example { + +SampleApp::SampleApp() + : VehicleApp(velocitas::IVehicleDataBrokerClient::createInstance("vehicledatabroker"), + velocitas::IPubSubClient::createInstance("SampleApp")) {} + +void SampleApp::onStart() { + auto seatService = + velocitas::SeatsServiceClientFactory::create(velocitas::Middleware::getInstance()); + + if (seatService != nullptr) { + velocitas::logger().info("Service available!"); + } +} + +} // namespace example diff --git a/test/cpp/app_seat_service_client/SampleApp.h b/test/cpp/app_seat_service_client/SampleApp.h new file mode 100644 index 00000000..46ea434a --- /dev/null +++ b/test/cpp/app_seat_service_client/SampleApp.h @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef VEHICLE_APP_SDK_EXT_GRPC_SERVICE_CLIENT_H +#define VEHICLE_APP_SDK_EXT_GRPC_SERVICE_CLIENT_H + +#include "sdk/Status.h" +#include "sdk/VehicleApp.h" + +#include +#include + +namespace example { + +class SampleApp : public velocitas::VehicleApp { +public: + SampleApp(); + + void onStart() override; +}; + +} // namespace example + +#endif // VEHICLE_APP_SDK_EXT_GRPC_SERVICE_CLIENT_H diff --git a/vehicle-model-lifecycle/README.md b/vehicle-model-lifecycle/README.md new file mode 100644 index 00000000..0fce49f5 --- /dev/null +++ b/vehicle-model-lifecycle/README.md @@ -0,0 +1,8 @@ +# Vehicle Signal Interface Support + +The "vehicle model lifecycle" / "vehicle signal interface support" component defines the process how the "code-level" Velocitas data model is generated from the signal catalogue referenced in the app's manifest. + +It consists roughly of the these steps: +* Getting the required dependencies needed for generating the model (e.g. the model generator itself) +* Making the referenced signal catalogue locally available (for the generator) +* Generating the signal interface model in the required programming language diff --git a/vehicle-model-lifecycle/requirements.txt b/vehicle-model-lifecycle/requirements.txt new file mode 100644 index 00000000..ee428502 --- /dev/null +++ b/vehicle-model-lifecycle/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/eclipse-velocitas/velocitas-lib.git@v0.0.2 diff --git a/vehicle-model-lifecycle/src/download_vspec.py b/vehicle-model-lifecycle/src/download_vspec.py index 9a5bf4fd..7c57d5e0 100644 --- a/vehicle-model-lifecycle/src/download_vspec.py +++ b/vehicle-model-lifecycle/src/download_vspec.py @@ -14,34 +14,16 @@ """Provides methods and functions to download vehicle model source files.""" -import json import os import re from typing import Any, Dict, List import requests +from velocitas_lib import get_app_manifest, get_project_cache_dir, get_workspace_dir FUNCTIONAL_INTERFACE_TYPE_KEY = "vehicle-signal-interface" -def require_env(name: str) -> str: - """Require and return an environment variable. - - Args: - name (str): The name of the variable. - - Raises: - ValueError: In case the environment variable is not set. - - Returns: - str: The value of the variable. - """ - var = os.getenv(name) - if not var: - raise ValueError(f"environment variable {var!r} not set!") - return var - - def is_uri(path: str) -> bool: """Check if the provided path is a URI. @@ -54,25 +36,7 @@ def is_uri(path: str) -> bool: return re.match(r"(\w+)\:\/\/(\w+)", path) is not None -def get_project_cache_dir() -> str: - """Return the project's cache directory. - - Returns: - str: The path to the project's cache directory. - """ - return require_env("VELOCITAS_CACHE_DIR") - - -def get_velocitas_workspace_dir() -> str: - """Return the project's workspace directory. - - Returns: - str: The path to the project's workspace directory. - """ - return require_env("VELOCITAS_WORKSPACE_DIR") - - -def download_file(uri: str, local_file_path: str): +def download_file(uri: str, local_file_path: str) -> None: """Download vspec file from the given URI to the project cache. Args: @@ -112,7 +76,7 @@ def get_legacy_model_src(app_manifest_dict: Dict[str, Any]) -> str: for key in possible_keys: if key in app_manifest_dict: - return app_manifest_dict[key]["src"] + return str(app_manifest_dict[key]["src"]) raise KeyError("App manifest does not contain a valid vehicle model!") @@ -159,10 +123,10 @@ def get_vehicle_signal_interface_src(interface: Dict[str, Any]) -> str: Returns: str: The URI of the source for the Vehicle Signal Interface. """ - return interface["config"]["src"] + return str(interface["config"]["src"]) -def main(app_manifest_dict: Dict[str, Any]): +def main(app_manifest_dict: Dict[str, Any]) -> None: """Entry point for downloading the vspec file for a vehicle-signal-interface. @@ -185,9 +149,7 @@ def main(app_manifest_dict: Dict[str, Any]): return vspec_src = get_vehicle_signal_interface_src(interfaces[0]) - local_vspec_path = os.path.join( - get_velocitas_workspace_dir(), os.path.normpath(vspec_src) - ) + local_vspec_path = os.path.join(get_workspace_dir(), os.path.normpath(vspec_src)) if is_uri(vspec_src): local_vspec_path = os.path.join(get_project_cache_dir(), "vspec.json") @@ -199,7 +161,4 @@ def main(app_manifest_dict: Dict[str, Any]): if __name__ == "__main__": - manifest_data_str = require_env("VELOCITAS_APP_MANIFEST") - app_manifest_dict = json.loads(manifest_data_str) - - main(app_manifest_dict) + main(get_app_manifest()) diff --git a/vehicle-model-lifecycle/src/generate_model.py b/vehicle-model-lifecycle/src/generate_model.py index 10ff7e0b..91140081 100644 --- a/vehicle-model-lifecycle/src/generate_model.py +++ b/vehicle-model-lifecycle/src/generate_model.py @@ -107,7 +107,7 @@ def install_model_if_required(language: str, model_path: str) -> None: subprocess.check_call([sys.executable, "-m", "pip", "install", model_path]) -def main(): +def main() -> None: """Main entry point for generation of vehicle models.""" cache_data = json.loads(require_env("VELOCITAS_CACHE_DATA")) diff --git a/vehicle-model-lifecycle/src/install_deps.py b/vehicle-model-lifecycle/src/install_deps.py index 3f4af5cd..aa7c34e7 100644 --- a/vehicle-model-lifecycle/src/install_deps.py +++ b/vehicle-model-lifecycle/src/install_deps.py @@ -58,7 +58,7 @@ def should_install_model_generator() -> bool: return False -def install_packages(): +def install_packages() -> None: """Install all required Python packages for the model generator and VSpec download.""" script_path = get_script_path() @@ -66,7 +66,7 @@ def install_packages(): model_gen_repo = require_env("modelGeneratorGitRepo") model_gen_version = require_env("modelGeneratorGitRef") - pip(["install", "-r", f"{script_path}/requirements.txt"]) + pip(["install", "-r", f"{script_path}/../requirements.txt"]) if should_install_model_generator(): pip(["install", f"git+{model_gen_repo}@{model_gen_version}"]) diff --git a/vehicle-model-lifecycle/src/requirements.txt b/vehicle-model-lifecycle/src/requirements.txt deleted file mode 100644 index 2c24336e..00000000 --- a/vehicle-model-lifecycle/src/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests==2.31.0 diff --git a/vehicle-model-lifecycle/test/integration/conftest.py b/vehicle-model-lifecycle/test/integration/conftest.py new file mode 100644 index 00000000..cad3b59a --- /dev/null +++ b/vehicle-model-lifecycle/test/integration/conftest.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Contributors to the Eclipse Foundation +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess + + +def pytest_sessionstart(session): + """ + Called after the Session object has been created and + before performing collection and entering the run test loop. + """ + + os.chdir(os.environ["VELOCITAS_TEST_ROOT"]) + + # FIXME: The C++ base image does not install conan globally + # but just for the vscode user, hence we have to download + # conan manually here. Can be removed once conan is installed + # globally. + if os.environ["VELOCITAS_TEST_LANGUAGE"] == "cpp": + subprocess.check_call(["python", "-m", "pip", "install", "conan==1.60.2"]) + + subprocess.check_call(["velocitas", "init", "-f", "-v"], stdin=subprocess.PIPE) diff --git a/vehicle-model-lifecycle/test/integration/test_integration.py b/vehicle-model-lifecycle/test/integration/test_integration.py new file mode 100644 index 00000000..047ee32b --- /dev/null +++ b/vehicle-model-lifecycle/test/integration/test_integration.py @@ -0,0 +1,11 @@ +import json +import os + + +def test_model_is_generated() -> None: + generated_model_path = json.load(open(".velocitas.json"))["variables"][ + "generatedModelPath" + ] + + assert os.path.exists(generated_model_path) + assert os.path.isdir(generated_model_path) diff --git a/vehicle-model-lifecycle/test/test_download_vspec.py b/vehicle-model-lifecycle/test/unit/test_download_vspec.py similarity index 91% rename from vehicle-model-lifecycle/test/test_download_vspec.py rename to vehicle-model-lifecycle/test/unit/test_download_vspec.py index 649b0ba8..cf843db9 100644 --- a/vehicle-model-lifecycle/test/test_download_vspec.py +++ b/vehicle-model-lifecycle/test/unit/test_download_vspec.py @@ -18,14 +18,12 @@ import pytest from test_lib import capture_stdout, mock_env -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "src")) from download_vspec import ( # noqa - download_file, get_legacy_model_src, is_proper_interface_type, is_uri, main, - require_env, ) vspec_300_uri = "https://github.com/COVESA/vehicle_signal_specification/releases/download/v3.0/vss_rel_3.0.json" # noqa @@ -45,13 +43,6 @@ def test_is_uri(): assert not is_uri("./local/path/to/file.file") -def test_download_file(): - local_file_path = "/tmp/mydownloadedfile" - download_file(vspec_300_uri, local_file_path) - - assert os.path.isfile(local_file_path) - - def test_get_legacy_model_src__camel_case_vehicle_model_key(): app_manifest_dict = {"vehicleModel": {"src": "foo"}} assert get_legacy_model_src(app_manifest_dict) == "foo" @@ -80,11 +71,6 @@ def test_proper_interface_type__correct_type(): assert is_proper_interface_type({"type": "vehicle-signal-interface"}) -def test_require_env__var_not_present__raises_error(): - with pytest.raises(ValueError): - require_env("foo") - - @pytest.mark.parametrize( "app_manifest", [ diff --git a/vehicle-model-lifecycle/test/test_generate_model.py b/vehicle-model-lifecycle/test/unit/test_generate_model.py similarity index 95% rename from vehicle-model-lifecycle/test/test_generate_model.py rename to vehicle-model-lifecycle/test/unit/test_generate_model.py index f40b47e7..fc7862a8 100644 --- a/vehicle-model-lifecycle/test/test_generate_model.py +++ b/vehicle-model-lifecycle/test/unit/test_generate_model.py @@ -15,7 +15,7 @@ import os import sys -sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "src")) from generate_model import get_model_output_dir # noqa diff --git a/vehicle-model-lifecycle/test/test_lib.py b/vehicle-model-lifecycle/test/unit/test_lib.py similarity index 100% rename from vehicle-model-lifecycle/test/test_lib.py rename to vehicle-model-lifecycle/test/unit/test_lib.py