From 968fc69e3ce2767f2a393f4d5d7235c3747dae9a Mon Sep 17 00:00:00 2001 From: nain-F49FF806 <126972030+nain-F49FF806@users.noreply.github.com> Date: Tue, 5 Mar 2024 07:27:01 +0100 Subject: [PATCH] feat(mediator): Containerise and package mediator (#1139) feat(mediator): Containerise and package mediator (#1139) Signed-off-by: Naian <126972030+nain-F49FF806@users.noreply.github.com> --- .github/workflows/mediator.pr.yml | 73 +++++++++++++++++-- aries/agents/rust/mediator/.env | 27 +++++++ aries/agents/rust/mediator/Dockerfile | 16 ++++ .../rust/mediator/Dockerfile.dockerignore | 1 + aries/agents/rust/mediator/README.md | 51 ++++++++++++- aries/agents/rust/mediator/compose.yaml | 27 +++++++ .../agents/rust/mediator/db-only.compose.yaml | 19 +++++ 7 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 aries/agents/rust/mediator/.env create mode 100644 aries/agents/rust/mediator/Dockerfile create mode 100644 aries/agents/rust/mediator/Dockerfile.dockerignore create mode 100644 aries/agents/rust/mediator/compose.yaml create mode 100644 aries/agents/rust/mediator/db-only.compose.yaml diff --git a/.github/workflows/mediator.pr.yml b/.github/workflows/mediator.pr.yml index 519688dd97..0c3f5824f6 100644 --- a/.github/workflows/mediator.pr.yml +++ b/.github/workflows/mediator.pr.yml @@ -13,18 +13,46 @@ on: env: DOCKER_BUILDKIT: 1 MAIN_BRANCH: main - URL_DOCKER_REGISTRY: ghcr.io - DOCKER_IMAGE_AGENCY: ghcr.io/absaoss/vcxagencynode/vcxagency-node:2.6.0 - DOCKER_IMAGE_POOL: ghcr.io/hyperledger/aries-vcx/indy_pool_localhost:1.15.0 - - DOCKER_REPO_LOCAL_VDRPROXY: vdrproxy RUST_TOOLCHAIN_VERSION: 1.74.1 - NODE_VERSION: 18.x jobs: + setup-variables: + runs-on: ubuntu-22.04 + outputs: + PUBLISH_VERSION: ${{ steps.run-info.outputs.publish-version }} + RELEASE: ${{ steps.run-info.outputs.release }} + PRERELEASE: ${{ steps.run-info.outputs.pre-release }} + BRANCH_NAME: ${{ steps.run-info.outputs.branch-name }} + IS_FORK: ${{ steps.run-info.outputs.is-fork }} + SKIP_CI: ${{ steps.skip-info.outputs.skip-ci }} + steps: + - name: "Git checkout" + uses: actions/checkout@v1 + - name: "Construct CI run-info" + id: run-info + uses: ./.github/actions/construct-run-info + - name: "Detect CI skip steps" + id: skip-info + uses: ./.github/actions/detect-skip-info + + check-variables: + runs-on: ubuntu-22.04 + needs: setup-variables + steps: + - name: "Print outputs" + run: | + echo "PUBLISH_VERSION ${{ needs.setup-variables.outputs.PUBLISH_VERSION }}" + echo "RELEASE ${{ needs.setup-variables.outputs.RELEASE }}" + echo "PRERELEASE ${{ needs.setup-variables.outputs.PRERELEASE }}" + echo "BRANCH_NAME ${{ needs.setup-variables.outputs.BRANCH_NAME }}" + echo "IS_FORK ${{ needs.setup-variables.outputs.IS_FORK }}" + echo "SKIP_CI ${{ needs.setup-variables.outputs.SKIP_CI }}" + test-integration-mediator: + needs: [ setup-variables ] + if: ${{ needs.setup-variables.outputs.SKIP_CI != 'true' }} runs-on: ubuntu-22.04 services: mysql: @@ -59,3 +87,36 @@ jobs: uses: ./.github/actions/upload-docker-logs with: name: "docker-services-${{ github.job }}" + + docker-mediator-build: + needs: [ test-integration-mediator, setup-variables ] + if: ${{ needs.setup-variables.outputs.SKIP_CI != 'true' }} + runs-on: ubuntu-22.04 + env: + DOCKER_IMAGE: mediator:${{ needs.setup-variables.outputs.BRANCH_NAME }} + outputs: + image-name: ${{ steps.meta.outputs.tags }} + steps: + - name: "Git checkout" + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: "Build and cache docker image" + uses: ./.github/actions/build-image + with: + docker-img: ${{ env.DOCKER_IMAGE }} + dockerfile-path: "aries/agents/rust/mediator/Dockerfile" + branch-name: ${{ needs.setup-variables.outputs.BRANCH_NAME }} + branch-main: ${{ env.MAIN_BRANCH }} + docker-repo-local-name: mediator + - name: "Publish versioned docker image" + uses: ./.github/actions/publish-image + with: + docker-img: ${{ env.DOCKER_IMAGE }} + publish-version: ${{ needs.setup-variables.outputs.PUBLISH_VERSION }} + env: + URL_DOCKER_REGISTRY: ghcr.io # Required by .github/actions/publish-image diff --git a/aries/agents/rust/mediator/.env b/aries/agents/rust/mediator/.env new file mode 100644 index 0000000000..66274abbc7 --- /dev/null +++ b/aries/agents/rust/mediator/.env @@ -0,0 +1,27 @@ +## Note: This file is read by `compose.yaml` for it's environment variables. +## So the services that docker-compose brings up will also use the below configuration. + +# Database (db) config + +## Root password can be autogenerated. It will be printed out on first run. +MYSQL_RANDOM_ROOT_PASSWORD=true + +## You can set it manually if you wish. Remember to unset MYSQL_RANDOM_ROOT_PASSWORD like below. +#MYSQL_RANDOM_ROOT_PASSWORD= +#MYSQL_ROOT_PASSWORD=poorpasswordavoid + +## Name of database to create on mysql init (first run). +MYSQL_DATABASE=mediator-persistence.db + +## User to generate on mysql init (first run). +MYSQL_USER=mediator +MYSQL_PASSWORD=mediator-pass + + +# Mediator (mediator) config + +## Full URL that mediator can use to connect to the database. +MYSQL_URL=mysql://mediator:mediator-pass@localhost:3306/mediator-persistence.db + +## This is the address at which the mediator will listen for connections. +ENDPOINT_ROOT=0.0.0.0:8005 diff --git a/aries/agents/rust/mediator/Dockerfile b/aries/agents/rust/mediator/Dockerfile new file mode 100644 index 0000000000..a4ffad07ab --- /dev/null +++ b/aries/agents/rust/mediator/Dockerfile @@ -0,0 +1,16 @@ +FROM rust as builder +RUN apt update && apt install -y libssl-dev libzmq3-dev cmake + +WORKDIR /usr/src/aries-vcx +COPY . . +RUN ls -lahF +RUN --mount=type=cache,target=./target \ + --mount=type=cache,target=${CARGO_HOME}/git \ + --mount=type=cache,target=${CARGO_HOME}/registry \ + cargo install --no-default-features --path=./aries/agents/rust/mediator/ --bin mediator + +FROM debian:bookworm-slim as mediator +RUN apt update && apt install -y libsodium23 libzmq5 && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/local/cargo/bin/mediator /usr/local/bin/mediator +CMD ["./usr/local/bin/mediator"] \ No newline at end of file diff --git a/aries/agents/rust/mediator/Dockerfile.dockerignore b/aries/agents/rust/mediator/Dockerfile.dockerignore new file mode 100644 index 0000000000..9f970225ad --- /dev/null +++ b/aries/agents/rust/mediator/Dockerfile.dockerignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/aries/agents/rust/mediator/README.md b/aries/agents/rust/mediator/README.md index d8c2966550..20ae684842 100644 --- a/aries/agents/rust/mediator/README.md +++ b/aries/agents/rust/mediator/README.md @@ -17,6 +17,8 @@ cargo build ## Usage +### Cargo + You can run and test the produced binaries using cargo. ```bash @@ -39,6 +41,53 @@ cargo run --bin mediator cargo test ``` +### Docker + +#### 1. You can build the docker image yourself or pull it from github + +`Dockerfile` is provided, to build the mediator image. +The image produced includes the mediator binary and required ssl libraries. + +```bash +# Note: Build context needs to include aries-vcx repository root. +docker build --tag mediator --file ./Dockerfile ../../../../ +``` + +Alternatively you can pull the latest prebuilt mediator image directly. + +```bash +docker pull ghcr.io/hyperledger/aries-vcx/mediator:main +``` + +#### 2. Use docker-compose with provided `compose.yaml` to quickly bring up mediator service along with mysql database + +```bash +# Note: Configuration can be customized using .env file, +# or by manually passing expected environment variables. +docker compose up -d +``` + +When you run the above, mediator and database containers are started on the same private network. +The configuration in .env file is used for database<->mediator connection parameters. +The mediator (but not the database) is additionally exposed on localhost:8005 by default for interaction. + +> [!IMPORTANT] +> While the database container uses a standard image, you will need to either build or pull the mediator image, +> as described in previous section. + +For regular development work on mediator, you may want to bring up only the database, +to test against local dev builds of mediator. + +```bash +docker compose -f db-only.compose.yaml up -d +``` + +To wind down services + +```bash +docker compose down +``` + ### Configurable Options Currently the mediator reads the following environment variables. @@ -47,7 +96,7 @@ Currently the mediator reads the following environment variables. `ENDPOINT_ROOT`: - **Description**: This is the address at which the mediator will listen for connections. - **Default**: "127.0.0.1:8005" -- **Usage**: `ENDPOINT_ROOT=127.0.0.1:3000 cargo run` +- **Usage**: `ENDPOINT_ROOT=0.0.0.0:3000` `MYSQL_URL`: - **Description**: MySQL url for the MYSQL database used for mediator persistence. diff --git a/aries/agents/rust/mediator/compose.yaml b/aries/agents/rust/mediator/compose.yaml new file mode 100644 index 0000000000..08e22da2bb --- /dev/null +++ b/aries/agents/rust/mediator/compose.yaml @@ -0,0 +1,27 @@ +services: + db: + image: mysql:latest + volumes: + - db_data:/var/lib/mysql + - ./migrations:/docker-entrypoint-initdb.d:z + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_RANDOM_ROOT_PASSWORD: ${MYSQL_RANDOM_ROOT_PASSWORD} + MYSQL_DATABASE: ${MYSQL_DATABASE} + MYSQL_USER: ${MYSQL_USER} + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + + mediator: + depends_on: + - db + image: mediator:latest + ports: + - "8005:8005" + environment: + MYSQL_URL: mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@db:3306/${MYSQL_DATABASE} + ENDPOINT_ROOT: ${ENDPOINT_ROOT} + +volumes: + db_data: + + diff --git a/aries/agents/rust/mediator/db-only.compose.yaml b/aries/agents/rust/mediator/db-only.compose.yaml new file mode 100644 index 0000000000..4dd5283b8a --- /dev/null +++ b/aries/agents/rust/mediator/db-only.compose.yaml @@ -0,0 +1,19 @@ +services: + db: + image: mysql:latest + volumes: + - db_data:/var/lib/mysql + - ./migrations:/docker-entrypoint-initdb.d:z + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_RANDOM_ROOT_PASSWORD: ${MYSQL_RANDOM_ROOT_PASSWORD} + MYSQL_DATABASE: ${MYSQL_DATABASE} + MYSQL_USER: ${MYSQL_USER} + MYSQL_PASSWORD: ${MYSQL_PASSWORD} + ports: + - "3306:3306" + +volumes: + db_data: + +