diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8f4c9cbd66..58f68b5d22 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/ubuntu/.devcontainer/base.Dockerfile # [Choice] Ubuntu version (use hirsuite or bionic on local arm64/Apple Silicon): hirsute, focal, bionic -ARG VARIANT="focal" +ARG VARIANT="jammy" FROM mcr.microsoft.com/vscode/devcontainers/base:${VARIANT} # [Optional] Uncomment this section to install additional OS packages. @@ -19,17 +19,17 @@ RUN apt-get update -y && apt-get install -y \ curl \ jq - # ======================================================================================================== # Update repository signing keys # -------------------------------------------------------------------------------------------------------- # Hyperledger RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9692C00E657DDE61 && \ # Sovrin - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 && \ + # bionic-security + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32 # ======================================================================================================== - # Plenum # - https://github.com/hyperledger/indy-plenum/issues/1546 # - Needed to pick up rocksdb=5.8.8 @@ -38,6 +38,12 @@ RUN echo "deb https://hyperledger.jfrog.io/artifactory/indy focal dev" >> /etc echo "deb https://repo.sovrin.org/deb bionic master" >> /etc/apt/sources.list && \ echo "deb https://repo.sovrin.org/sdk/deb bionic master" >> /etc/apt/sources.list +# Kim's temp repo +# TODO: +# - Remove dependency on Kim's temp repo. +# - Change this to official repo +RUN echo "deb [trusted=yes] http://209.141.41.82:8000/ packagedir/" >> /etc/apt/sources.list + RUN apt-get update -y && apt-get install -y \ # Python python3-pip \ @@ -63,6 +69,11 @@ RUN apt-get update -y && apt-get install -y \ ursa=0.3.2-1 \ # Indy SDK libindy=1.15.0~1625-bionic \ + # Kim's updated packages + # - TODO: Remove dependency on Kim's temp repo. + python3-sortedcontainers \ + python3-rlp \ + python3-ioflo \ # Need to move libursa.so to parent dir && mv /usr/lib/ursa/* /usr/lib && rm -rf /usr/lib/ursa @@ -78,4 +89,4 @@ RUN pip3 install -U \ # install fpm -RUN gem install --no-document rake dotenv:2.8.1 fpm:1.14.2 \ No newline at end of file +RUN gem install --no-document rake dotenv:2.8.1 fpm:1.15.0 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 52896fe80f..596285d4dc 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,7 @@ "dockerfile": "Dockerfile", // Update 'VARIANT' to pick an Ubuntu version: hirsute, focal, bionic // Use hirsute or bionic on local arm64/Apple Silicon. - "args": { "VARIANT": "focal" } + "args": { "VARIANT": "jammy" } }, // Set *default* container specific settings.json values on container create. @@ -29,4 +29,4 @@ // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" -} \ No newline at end of file +} diff --git a/.github/workflows/PR.yaml b/.github/workflows/PR.yaml index 1c1d7a41a4..50113581a0 100644 --- a/.github/workflows/PR.yaml +++ b/.github/workflows/PR.yaml @@ -7,6 +7,7 @@ on: branches: - main + - ubuntu-22.04 jobs: workflow-setup: @@ -28,7 +29,7 @@ jobs: uses: actions/checkout@v4 - name: setup id: setup - uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v1 + uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v2 - name: testsNeeded id: testsNeeded uses: dorny/paths-filter@v3 @@ -44,12 +45,12 @@ jobs: name: Lint needs: [workflow-setup] if: ${{ needs.workflow-setup.outputs.testsNeeded == 'true' }} - uses: hyperledger/indy-shared-gha/.github/workflows/lint.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/lint.yaml@v2 build-docker-image: name: Create Builder Image needs: [workflow-setup, lint] - uses: hyperledger/indy-shared-gha/.github/workflows/buildimage.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/buildimage.yaml@v2 with: CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build @@ -58,7 +59,7 @@ jobs: build_packages: name: Build Packages needs: [workflow-setup, build-docker-image] - uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v2 with: DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} @@ -80,4 +81,4 @@ jobs: needs: [workflow-setup, plenum_tests] if: ${{ needs.workflow-setup.outputs.testsNeeded == 'false' || success() }} steps: - - run: 'echo "Just a status Check (Always true, when executed) for branch protection rules(blocks merging while test are running and if tests fail)." ' \ No newline at end of file + - run: 'echo "Just a status Check (Always true, when executed) for branch protection rules(blocks merging while test are running and if tests fail)." ' diff --git a/.github/workflows/Push.yaml b/.github/workflows/Push.yaml index a6fa151ef2..e5101b28a6 100644 --- a/.github/workflows/Push.yaml +++ b/.github/workflows/Push.yaml @@ -3,6 +3,7 @@ on: push: branches: - main + - ubuntu-22.04 paths: - '**.py' - '.github/**' @@ -28,16 +29,16 @@ jobs: uses: actions/checkout@v4 - name: setup id: setup - uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v1 + uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v2 lint: name: Lint - uses: hyperledger/indy-shared-gha/.github/workflows/lint.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/lint.yaml@v2 build-docker-image: name: Create Builder Image needs: [workflow-setup, lint] - uses: hyperledger/indy-shared-gha/.github/workflows/buildimage.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/buildimage.yaml@v2 with: CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }} DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build @@ -46,7 +47,7 @@ jobs: build_packages: name: Build Packages needs: [workflow-setup, build-docker-image] - uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v2 with: DOCKER_IMAGE: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} @@ -66,7 +67,7 @@ jobs: name: Publish Artifacts needs: [workflow-setup, plenum_tests] if: needs.workflow-setup.outputs.publish == 'true' - uses: hyperledger/indy-shared-gha/.github/workflows/publish_artifacts.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/publish_artifacts.yaml@v2 with: COMPONENT: 'dev' UBUNTU_VERSION: ${{ needs.workflow-setup.outputs.UBUNTU_VERSION }} diff --git a/.github/workflows/build/Dockerfile.ubuntu-2204 b/.github/workflows/build/Dockerfile.ubuntu-2204 new file mode 100644 index 0000000000..0de89a80f1 --- /dev/null +++ b/.github/workflows/build/Dockerfile.ubuntu-2204 @@ -0,0 +1,42 @@ +FROM ubuntu:22.04 + +ARG uid=1000 +ARG user=indy + +RUN apt-get update -y && apt-get install -y \ + # common stuff + git \ + apt-transport-https \ + apt-utils\ + wget\ + curl\ + jq\ + gnupg + +# ======================================================================================================== +# Update repository signing keys +# -------------------------------------------------------------------------------------------------------- + # Hyperledger +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 9692C00E657DDE61 && \ + # Sovrin + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CE7709D068DB5E88 +# ======================================================================================================== + +# Plenum +# - https://github.com/hyperledger/indy-plenum/issues/1546 +# - Needed to pick up rocksdb=5.8.8 +#RUN echo "deb https://hyperledger.jfrog.io/artifactory/indy focal dev" >> /etc/apt/sources.list && \ +# echo "deb https://repo.sovrin.org/deb bionic master" >> /etc/apt/sources.list && \ +# echo "deb https://repo.sovrin.org/sdk/deb bionic master" >> /etc/apt/sources.list + +# TODO change this to official repo +RUN echo "deb [trusted=yes] http://209.141.41.82:8000/ packagedir/" >> /etc/apt/sources.list + + +RUN apt-get update -y && apt-get install -y \ + # Python + python3-pip \ + rubygems && \ + gem install --no-document dotenv:2.8.1 fpm:1.15.0 && \ + pip3 install Cython==0.29.36 + diff --git a/.github/workflows/publishRelease.yaml b/.github/workflows/publishRelease.yaml index 304ad68241..3d69cf4dbc 100644 --- a/.github/workflows/publishRelease.yaml +++ b/.github/workflows/publishRelease.yaml @@ -34,12 +34,12 @@ jobs: uses: actions/checkout@v4 - name: get-release-info id: get-release-info - uses: hyperledger/indy-shared-gha/.github/actions/get-release-info@v1 + uses: hyperledger/indy-shared-gha/.github/actions/get-release-info@v2 with: versionString: "${{ github.event.head_commit.message }}" - name: workflow-setup id: workflow-setup - uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v1 + uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v2 createRelease: name: Create Release @@ -111,7 +111,7 @@ jobs: name: Publish Artifacts needs: [release-infos, createRelease] if: needs.release-infos.outputs.isVersionBump == 'true' && needs.release-infos.outputs.publish == 'true' - uses: hyperledger/indy-shared-gha/.github/workflows/publish_artifacts.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/publish_artifacts.yaml@v2 with: COMPONENT: ${{ needs.release-infos.outputs.component }} UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }} @@ -124,7 +124,7 @@ jobs: convertPyVersion: name: "Convert to python version flavour" needs: [release-infos, publish_artifacts] - uses: hyperledger/indy-shared-gha/.github/workflows/pyVersionConversion.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/pyVersionConversion.yaml@v2 with: VERSIONTAG: ${{ needs.release-infos.outputs.VERSIONTAG }} diff --git a/.github/workflows/releasepr.yaml b/.github/workflows/releasepr.yaml index a44e909921..e6b78f47ce 100644 --- a/.github/workflows/releasepr.yaml +++ b/.github/workflows/releasepr.yaml @@ -26,24 +26,24 @@ jobs: uses: actions/checkout@v4 - name: get-release-info id: get-release-info - uses: hyperledger/indy-shared-gha/.github/actions/get-release-info@v1 + uses: hyperledger/indy-shared-gha/.github/actions/get-release-info@v2 with: versionString: "${{ github.event.pull_request.body }}" - name: workflow-setup id: workflow-setup - uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v1 + uses: hyperledger/indy-shared-gha/.github/actions/workflow-setup@v2 lint: name: Lint needs: [release-infos] if: needs.release-infos.outputs.isVersionBump == 'true' - uses: hyperledger/indy-shared-gha/.github/workflows/lint.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/lint.yaml@v2 build-docker-image: name: Create Builder Image needs: [release-infos, lint] if: needs.release-infos.outputs.isVersionBump == 'true' - uses: hyperledger/indy-shared-gha/.github/workflows/buildimage.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/buildimage.yaml@v2 with: CACHE_KEY_BUILD: ${{ needs.release-infos.outputs.CACHE_KEY_BUILD }} DOCKER_IMAGE: ghcr.io/${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build @@ -53,7 +53,7 @@ jobs: name: Build Packages needs: [release-infos, build-docker-image] if: needs.release-infos.outputs.isVersionBump == 'true' - uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v1 + uses: hyperledger/indy-shared-gha/.github/workflows/buildpackages.yaml@v2 with: DOCKER_IMAGE: ghcr.io/${{ needs.release-infos.outputs.GITHUB_REPOSITORY_NAME }}/plenum-build:${{ needs.release-infos.outputs.UBUNTU_VERSION }} UBUNTU_VERSION: ${{ needs.release-infos.outputs.UBUNTU_VERSION }} diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml index 932a06010c..41cc936741 100644 --- a/.github/workflows/tag.yaml +++ b/.github/workflows/tag.yaml @@ -21,12 +21,12 @@ jobs: fetch-depth: 0 - name: extract branch id: get-branch - uses: hyperledger/indy-shared-gha/.github/actions/branch-from-tag@v1 + uses: hyperledger/indy-shared-gha/.github/actions/branch-from-tag@v2 with: tag: ${{ github.ref }} - name: get-release-info id: get-release-info - uses: hyperledger/indy-shared-gha/.github/actions/get-release-info@v1 + uses: hyperledger/indy-shared-gha/.github/actions/get-release-info@v2 with: versionString: "${{ github.ref }}" diff --git a/build-scripts/ubuntu-2204/README.md b/build-scripts/ubuntu-2204/README.md new file mode 100644 index 0000000000..ffa70fba59 --- /dev/null +++ b/build-scripts/ubuntu-2204/README.md @@ -0,0 +1,30 @@ +### Build indy-plenum using docker + +``` +./build-indy-plenum-docker.sh +``` +Built package is placed in a docker volume `indy-plenum-deb-u1604`. + +### Build indy-plenum + +``` +./build-indy-plenum.sh +``` + +Built package is placed in the `output-path` folder. + +### Build 3rd-party dependencies using docker + +``` +./build-3rd-parties-docker.sh +``` + +Built packages are placed in a docker volume `indy-plenum-deb-u1604`. + +### Build 3rd-party dependencies + +``` +./build-3rd-parties-docker.sh +``` + +Built packages are placed in the `output-path` folder. diff --git a/build-scripts/ubuntu-2204/build-3rd-parties-docker.sh b/build-scripts/ubuntu-2204/build-3rd-parties-docker.sh new file mode 100755 index 0000000000..7d09e753cb --- /dev/null +++ b/build-scripts/ubuntu-2204/build-3rd-parties-docker.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -x +set -e + + +PKG_SOURCE_PATH=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../../ + +if [ -z "$2" ]; then + CMD="/input/build-scripts/ubuntu-2204/build-3rd-parties.sh /output" +else + CMD="$2" +fi + +PKG_NAME=indy-plenum +IMAGE_NAME="${PKG_NAME}-build-u2204" +OUTPUT_VOLUME_NAME="${1:-"${PKG_NAME}-deb-u2204"}" + +docker build -t "${PKG_NAME}-build-u2204" -f $PKG_SOURCE_PATH/.github/workflows/build/Dockerfile.ubuntu-2204 . +docker volume create --name "${OUTPUT_VOLUME_NAME}" + +docker run \ + -i \ + --rm \ + -v "${PKG_SOURCE_PATH}:/input" \ + -v "${OUTPUT_VOLUME_NAME}:/output" \ + "${IMAGE_NAME}" \ + $CMD diff --git a/build-scripts/ubuntu-2204/build-3rd-parties.sh b/build-scripts/ubuntu-2204/build-3rd-parties.sh new file mode 100755 index 0000000000..4900925d14 --- /dev/null +++ b/build-scripts/ubuntu-2204/build-3rd-parties.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +set -e +set -x + +# Ensure OUTPUT_PATH is defined as a fully qualified path +pushd $(dirname $(realpath "$0")) +OUTPUT_PATH=${1:-.} +if [ ! -d "${OUTPUT_PATH}" ]; then + mkdir -p ${OUTPUT_PATH} +fi +OUTPUT_PATH=$(realpath ${OUTPUT_PATH}) +popd + +wheel2debconf="$(dirname "$(realpath "$0")")"/wheel2deb.yml + +function build_rocksdb_deb { + VERSION=$1 + VERSION_TAG="rocksdb-$VERSION" + #Install rocksdb requirements (libsnappy lbz2 llz4) + apt update && apt install -y libsnappy-dev libbz2-dev liblz4-dev zlib1g-dev libgflags-dev + git clone https://github.com/evernym/rocksdb.git /tmp/rocksdb + scriptpath="$(dirname "$(realpath "$0")")"/make_rocksdb.sh + cd /tmp/rocksdb + git checkout $VERSION_TAG + cp $scriptpath /tmp/rocksdb/build_tools/make_package.sh + sed -i 's/-m rocksdb@fb.com/-m "Hyperledger "/g' \ + ./build_tools/make_package.sh + PORTABLE=1 EXTRA_CFLAGS="-fPIC" EXTRA_CXXFLAGS="-fPIC" ./build_tools/make_package.sh $VERSION + # Install it in the system as it is needed by python-rocksdb. + make install + cd - + cp /tmp/rocksdb/package/rocksdb_${VERSION}_amd64.deb $OUTPUT_PATH + rm -rf /tmp/rocksdb +} + +function build_ioflo_deb { + VERSION=$1 + + git clone https://github.com/reflectivedevelopment/ioflo.git /tmp/ioflo + pushd /tmp/ioflo + git checkout $VERSION + + python3 setup.py bdist_wheel + pushd dist + wheel2deb --config ${wheel2debconf} + mv output/*.deb ${OUTPUT_PATH} + + popd + rm -rf /tmp/ioflo + popd +} + +function build_from_pypi_fpm { + PACKAGE_NAME=$1 + + if [ -z $2 ]; then + PACKAGE_VERSION="" + # Get the most recent package version from PyPI to be included in the package name of the Debian artifact + curl -X GET "https://pypi.org/pypi/${PACKAGE_NAME}/json" > "${PACKAGE_NAME}.json" + PACKAGE_VERSION="==$(cat "${PACKAGE_NAME}.json" | jq --raw-output '.info.version')" + rm "${PACKAGE_NAME}.json" + else + PACKAGE_VERSION="==$2" + fi + POSTINST_TMP=postinst-${PACKAGE_NAME} + PREREM_TMP=prerm-${PACKAGE_NAME} + cp postinst ${POSTINST_TMP} + cp prerm ${PREREM_TMP} + if [[ ${PACKAGE_NAME} =~ ^python-* ]]; then + PACKAGE_NAME_TMP="${PACKAGE_NAME/python-/}" + else + PACKAGE_NAME_TMP=$PACKAGE_NAME + fi + sed -i 's/{package_name}/python3-'${PACKAGE_NAME_TMP}'/' ${POSTINST_TMP} + sed -i 's/{package_name}/python3-'${PACKAGE_NAME_TMP}'/' ${PREREM_TMP} + + if [ -z $3 ]; then + fpm --input-type "python" \ + --output-type "deb" \ + --architecture "amd64" \ + --verbose \ + --python-package-name-prefix "python3"\ + --python-bin "/usr/bin/python3" \ + --exclude "*.pyc" \ + --exclude "*.pyo" \ + --maintainer "Hyperledger " \ + --after-install ${POSTINST_TMP} \ + --before-remove ${PREREM_TMP} \ + --package ${OUTPUT_PATH} \ + ${PACKAGE_NAME}${PACKAGE_VERSION} + else + fpm --input-type "python" \ + --output-type "deb" \ + --architecture "amd64" \ + --python-setup-py-arguments "${3}" \ + --verbose \ + --python-package-name-prefix "python3"\ + --python-bin "/usr/bin/python3" \ + --exclude "*.pyc" \ + --exclude "*.pyo" \ + --maintainer "Hyperledger " \ + --after-install ${POSTINST_TMP} \ + --before-remove ${PREREM_TMP} \ + --package ${OUTPUT_PATH} \ + ${PACKAGE_NAME}${PACKAGE_VERSION} + + # --python-pip "$(which pip)" \ + # ERROR: download_if_necessary': Unexpected directory layout after easy_install. Maybe file a bug? The directory is /tmp/package-python-build-c42d23109dcca1e98d9f430a04fe79a815f10d8ed7a719633aa969424f94 (RuntimeError) + fi + + rm ${POSTINST_TMP} + rm ${PREREM_TMP} +} + +function build_from_pypi_wheel { + PACKAGE_NAME=$1 + + if [ -z $2 ]; then + PACKAGE_VERSION="" + # Get the most recent package version from PyPI to be included in the package name of the Debian artifact + curl -X GET "https://pypi.org/pypi/${PACKAGE_NAME}/json" > "${PACKAGE_NAME}.json" + PACKAGE_VERSION="==$(cat "${PACKAGE_NAME}.json" | jq --raw-output '.info.version')" + rm "${PACKAGE_NAME}.json" + else + PACKAGE_VERSION="==$2" + fi + + rm -rvf /tmp/wheel + mkdir /tmp/wheel + pushd /tmp/wheel + pip3 wheel ${PACKAGE_NAME}${PACKAGE_VERSION} + # Can't build cytoolz using wheel for rlp, but can't build rlp with fpm + rm -f /tmp/wheel/cytoolz* + wheel2deb --config ${wheel2debconf} + popd + mv /tmp/wheel/output/*.deb ${OUTPUT_PATH} + rm -rvf /tmp/wheel +} + + +# TODO duplicates list from Jenkinsfile.cd +SCRIPT_PATH="${BASH_SOURCE[0]}" +pushd `dirname ${SCRIPT_PATH}` >/dev/null + +# Install any python requirements needed for the builds. +pip install -r requirements.txt +pip3 install wheel2deb && apt-get install -y debhelper +apt-get install -y cython3 + +# Build rocksdb at first +### Can be removed once the code has been updated to run with rocksdb v. 5.17 +### Issue 1551: Update RocksDB; https://github.com/hyperledger/indy-plenum/issues/1551 +build_rocksdb_deb 5.8.8 + +#### PyZMQCommand +build_from_pypi_fpm pyzmq 22.3.0 --zmq=bundled + +##### install_requires +build_from_pypi_wheel base58 +### Needs to be pinned to 3.10.1 because from v4.0.0 the package name ends in python3-importlib-metadata_0.0.0_amd64.deb +### https://github.com/hyperledger/indy-plenum/runs/4166593170?check_suite_focus=true#step:5:5304 +build_from_pypi_wheel importlib-metadata 3.10.1 +build_ioflo_deb 2.0.3 +build_from_pypi_wheel jsonpickle +build_from_pypi_wheel leveldb +build_from_pypi_wheel libnacl 1.6.1 +build_from_pypi_wheel msgpack-python +build_from_pypi_wheel orderedset +build_from_pypi_wheel packaging 21.3 +build_from_pypi_wheel portalocker 2.7.0 +build_from_pypi_wheel prompt-toolkit 3.0.18 +build_from_pypi_fpm psutil +build_from_pypi_wheel pympler 0.8 +build_from_pypi_wheel python-dateutil +build_from_pypi_wheel python-rocksdb +build_from_pypi_wheel python-ursa 0.1.1 +build_from_pypi_wheel rlp 2.0.0 +build_from_pypi_fpm cytoolz 0.12.3 +build_from_pypi_wheel semver 2.13.0 +build_from_pypi_wheel sha3 +build_from_pypi_wheel six +build_from_pypi_wheel sortedcontainers 2.1.0 +build_from_pypi_wheel ujson 1.33 + +rm -vf ${OUTPUT_PATH}/python3-setuptools*.deb diff --git a/build-scripts/ubuntu-2204/build-indy-plenum-docker.sh b/build-scripts/ubuntu-2204/build-indy-plenum-docker.sh new file mode 100755 index 0000000000..c733d94ec0 --- /dev/null +++ b/build-scripts/ubuntu-2204/build-indy-plenum-docker.sh @@ -0,0 +1,31 @@ +#!/bin/bash -xe + +PKG_SOURCE_PATH="$1" +VERSION="$2" +PKG_NAME=indy-plenum +IMAGE_NAME="${PKG_NAME}-build-u2204" +OUTPUT_VOLUME_NAME="${3:-"${PKG_NAME}-deb-u2204"}" +PACKAGE_VERSION="${4:-$VERSION}" + +if [[ (-z "${PKG_SOURCE_PATH}") || (-z "${VERSION}") ]]; then + echo "Usage: $0 [ [package-version]]" + exit 1; +fi + +if [ -z "$5" ]; then + CMD="/input/build-scripts/ubuntu-2204/build-${PKG_NAME}.sh /input ${VERSION} /output ${PACKAGE_VERSION}" +else + CMD="$5" +fi + +docker build -t "${IMAGE_NAME}" -f ../../.github/workflows/build/Dockerfile.ubuntu-2204 . +docker volume create --name "${OUTPUT_VOLUME_NAME}" + +docker run \ + -i \ + --rm \ + -v "${PKG_SOURCE_PATH}:/input" \ + -v "${OUTPUT_VOLUME_NAME}:/output" \ + -e PKG_NAME="${PKG_NAME}" \ + "${IMAGE_NAME}" \ + $CMD diff --git a/build-scripts/ubuntu-2204/build-plenum.sh b/build-scripts/ubuntu-2204/build-plenum.sh new file mode 100755 index 0000000000..8ec1c3dfb5 --- /dev/null +++ b/build-scripts/ubuntu-2204/build-plenum.sh @@ -0,0 +1,45 @@ +#!/bin/bash -xe + +INPUT_PATH=$1 +VERSION=$2 +OUTPUT_PATH=${3:-.} +PACKAGE_VERSION=${4:-$VERSION} + +PACKAGE_NAME=indy-plenum + +# copy the sources to a temporary folder +TMP_DIR=$(mktemp -d) +cp -r ${INPUT_PATH}/. ${TMP_DIR} + +# prepare the sources +cd ${TMP_DIR}/build-scripts/ubuntu-2204 +./prepare-package.sh ${TMP_DIR} plenum ${VERSION} debian-packages + +sed -i 's/{package_name}/'${PACKAGE_NAME}'/' "postinst" +sed -i 's/{package_name}/'${PACKAGE_NAME}'/' "prerm" + +fpm --input-type "python" \ + --output-type "deb" \ + --architecture "amd64" \ + --depends "python3-pyzmq (= 22.3.0)" \ + --depends "rocksdb (=5.8.8)"\ + --depends "ursa (= 0.3.2-1)"\ + --verbose \ + --python-package-name-prefix "python3"\ + --python-bin "/usr/bin/python3" \ + --exclude "usr/local/lib/python3.8/dist-packages/data" \ + --exclude "usr/local/bin" \ + --exclude "*.pyc" \ + --exclude "*.pyo" \ + --maintainer "Hyperledger " \ + --after-install "postinst" \ + --before-remove "prerm" \ + --name ${PACKAGE_NAME} \ + --version ${PACKAGE_VERSION} \ + --package ${OUTPUT_PATH} \ + ${TMP_DIR} + + # --python-pip "$(which pip)" \ + # ERROR: download_if_necessary': Unexpected directory layout after easy_install. Maybe file a bug? The directory is /tmp/package-python-build-c42d23109dcca1e98d9f430a04fe79a815f10d8ed7a719633aa969424f94 (RuntimeError) + +rm -rf ${TMP_DIR} diff --git a/build-scripts/ubuntu-2204/make_rocksdb.sh b/build-scripts/ubuntu-2204/make_rocksdb.sh new file mode 100644 index 0000000000..1c344e1dac --- /dev/null +++ b/build-scripts/ubuntu-2204/make_rocksdb.sh @@ -0,0 +1,138 @@ +# shellcheck disable=SC1113 +#/usr/bin/env bash + +set -e + +function log() { + echo "[+] $1" +} + +function fatal() { + echo "[!] $1" + exit 1 +} + +function platform() { + local __resultvar=$1 + if [[ -f "/etc/yum.conf" ]]; then + eval $__resultvar="centos" + elif [[ -f "/etc/dpkg/dpkg.cfg" ]]; then + eval $__resultvar="ubuntu" + else + fatal "Unknwon operating system" + fi +} +platform OS + +function package() { + if [[ $OS = "ubuntu" ]]; then + if dpkg --get-selections | grep --quiet $1; then + log "$1 is already installed. skipping." + else + # shellcheck disable=SC2068 + apt-get install $@ -y + fi + elif [[ $OS = "centos" ]]; then + if rpm -qa | grep --quiet $1; then + log "$1 is already installed. skipping." + else + # shellcheck disable=SC2068 + yum install $@ -y + fi + fi +} + +function detect_fpm_output() { + if [[ $OS = "ubuntu" ]]; then + export FPM_OUTPUT=deb + elif [[ $OS = "centos" ]]; then + export FPM_OUTPUT=rpm + fi +} +detect_fpm_output + +function gem_install() { + if gem list | grep --quiet $1; then + log "$1 is already installed. skipping." + else + # shellcheck disable=SC2068 + gem install $@ + fi +} + +function main() { + if [[ $# -ne 1 ]]; then + fatal "Usage: $0 " + else + log "using rocksdb version: $1" + fi + + if [[ -d /vagrant ]]; then + if [[ $OS = "ubuntu" ]]; then + package g++-4.8 + export CXX=g++-4.8 + + # the deb would depend on libgflags2, but the static lib is the only thing + # installed by make install + package libgflags-dev + + package ruby-all-dev + elif [[ $OS = "centos" ]]; then + pushd /etc/yum.repos.d + if [[ ! -f /etc/yum.repos.d/devtools-1.1.repo ]]; then + wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo + fi + package devtoolset-1.1-gcc --enablerepo=testing-1.1-devtools-6 + package devtoolset-1.1-gcc-c++ --enablerepo=testing-1.1-devtools-6 + export CC=/opt/centos/devtoolset-1.1/root/usr/bin/gcc + export CPP=/opt/centos/devtoolset-1.1/root/usr/bin/cpp + export CXX=/opt/centos/devtoolset-1.1/root/usr/bin/c++ + export PATH=$PATH:/opt/centos/devtoolset-1.1/root/usr/bin + popd + if ! rpm -qa | grep --quiet gflags; then + rpm -i https://github.com/schuhschuh/gflags/releases/download/v2.1.0/gflags-devel-2.1.0-1.amd64.rpm + fi + + package ruby + package ruby-devel + package rubygems + package rpm-build + fi + fi + gem_install fpm + + make static_lib + make install INSTALL_PATH=package + + cd package + + LIB_DIR=lib + if [[ -z "$ARCH" ]]; then + ARCH=$(getconf LONG_BIT) + fi + if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then + mv lib lib64 + LIB_DIR=lib64 + fi + + fpm \ + -s dir \ + -t $FPM_OUTPUT \ + -n rocksdb \ + -v $1 \ + --prefix /usr \ + --url http://rocksdb.org/ \ + -m rocksdb@fb.com \ + --license BSD \ + --vendor Facebook \ + --depends "libgflags-dev" \ + --depends "libsnappy-dev" \ + --depends "zlib1g-dev" \ + --depends "libbz2-dev" \ + --depends "liblz4-dev" \ + --description "RocksDB is an embeddable persistent key-value store for fast storage." \ + include $LIB_DIR +} + +# shellcheck disable=SC2068 +main $@ \ No newline at end of file diff --git a/build-scripts/ubuntu-2204/postinst b/build-scripts/ubuntu-2204/postinst new file mode 100644 index 0000000000..31b5145050 --- /dev/null +++ b/build-scripts/ubuntu-2204/postinst @@ -0,0 +1,8 @@ +#!/bin/bash + +# Automatically added from template: +if which py3compile >/dev/null 2>&1; then + py3compile -O -p {package_name} /usr/local/lib/python3.8/dist-packages/ +fi + +# End automatically added section diff --git a/build-scripts/ubuntu-2204/prepare-package.sh b/build-scripts/ubuntu-2204/prepare-package.sh new file mode 100755 index 0000000000..b08c8f3100 --- /dev/null +++ b/build-scripts/ubuntu-2204/prepare-package.sh @@ -0,0 +1,45 @@ +#!/bin/bash -ex + +if [ "$1" = "--help" ] ; then + echo "Usage: $0 " + echo " - Set to 'debian-packages' when preparing deb packages, and 'python-packages' when preparing PyPi packages." + exit 0 +fi + +repo="$1" +module_name="$2" +version_dotted="$3" +distro_packages="$4" + +BUMP_SH_SCRIPT="bump_version.sh" +GENERATE_MANIFEST_SCRIPT="generate_manifest.sh" + +pushd $repo + +echo -e "\nSetting version to $version_dotted" +bash -ex $BUMP_SH_SCRIPT $version_dotted +cat $module_name/__version__.json + +echo -e "\nGenerating manifest" +bash -ex $GENERATE_MANIFEST_SCRIPT +cat $module_name/__manifest__.json + +if [ "$distro_packages" = "debian-packages" ]; then + # Only used for the deb package builds, NOT for the PyPi package builds. + # Update the package names to match the versions that are pre-installed on the os. + echo -e "\nAdapt the dependencies for the Canonical archive" + #### ToDo adjust packages for the Cannonical archive for Ubuntu 20.04 (focal) + # sed -i "s~ujson==1.33~ujson==1.33-1build1~" setup.py + # sed -i "s~prompt_toolkit==0.57~prompt_toolkit==0.57-1~" setup.py + # sed -i "s~msgpack-python==0.4.6~msgpack==0.4.6-1build1~" setup.py + sed -i "s~msgpack-python~msgpack~" setup.py +elif [ "$distro_packages" = "python-packages" ]; then + echo -e "\nNo adaption of dependencies for python packages" +else + echo -e "\nNo distribution specified. Please, specify distribution as 'debian-packages' or 'python-packages'." + exit 1 +fi + +popd + +echo -e "\nFinished preparing $repo for publishing\n" \ No newline at end of file diff --git a/build-scripts/ubuntu-2204/prerm b/build-scripts/ubuntu-2204/prerm new file mode 100644 index 0000000000..40af9b22a9 --- /dev/null +++ b/build-scripts/ubuntu-2204/prerm @@ -0,0 +1,7 @@ +#!/bin/bash + +# Automatically added from template: + +dpkg -L {package_name} | perl -ne 's,/([^/]*)\.py$,/__pycache__/\1.*, or next; unlink $_ or die $! foreach glob($_)' + +# End automatically added section \ No newline at end of file diff --git a/build-scripts/ubuntu-2204/requirements.txt b/build-scripts/ubuntu-2204/requirements.txt new file mode 100644 index 0000000000..a7c8cc5b85 --- /dev/null +++ b/build-scripts/ubuntu-2204/requirements.txt @@ -0,0 +1,2 @@ +# Requirements for buuilding the python3-pyzmq_22.3.x_amd64.deb package. +packaging \ No newline at end of file diff --git a/build-scripts/ubuntu-2204/wheel2deb.yml b/build-scripts/ubuntu-2204/wheel2deb.yml new file mode 100644 index 0000000000..6673473819 --- /dev/null +++ b/build-scripts/ubuntu-2204/wheel2deb.yml @@ -0,0 +1,6 @@ +# change setting for all wheels +.: + maintainer_name: "Hyperledger" + maintainer_email: "hyperledger-indy@lists.hyperledger.org" + extended_desc: "Hyperledger package for indy-plenum" + version_template: "{upstream_version}" diff --git a/plenum/__metadata__.py b/plenum/__metadata__.py index dcd4db3b35..a1056ff532 100644 --- a/plenum/__metadata__.py +++ b/plenum/__metadata__.py @@ -4,7 +4,7 @@ import os import json from typing import Any -import collections.abc +from collections.abc import Iterable from common.version import PlenumVersion, InvalidVersionError @@ -21,7 +21,7 @@ def load_version(version_file: str = VERSION_FILE) -> PlenumVersion: with open(version_file, 'r') as _f: version = json.load(_f) - if not isinstance(version, collections.abc.Iterable): + if not isinstance(version, Iterable): raise InvalidVersionError( "Failed to load from {}: data '{}' is not iterable" .format(version_file, version) diff --git a/plenum/common/util.py b/plenum/common/util.py index ea44ef9206..432fb6fa2b 100644 --- a/plenum/common/util.py +++ b/plenum/common/util.py @@ -538,7 +538,7 @@ def getCallableName(callable: Callable): def updateNestedDict(d, u, nestedKeysToUpdate=None): for k, v in u.items(): - if isinstance(v, collections.Mapping) and \ + if isinstance(v, collections.abc.Mapping) and \ (not nestedKeysToUpdate or k in nestedKeysToUpdate): r = updateNestedDict(d.get(k, {}), v) d[k] = r diff --git a/plenum/persistence/req_id_to_txn.py b/plenum/persistence/req_id_to_txn.py index fb58c5865e..e60ebb2b31 100644 --- a/plenum/persistence/req_id_to_txn.py +++ b/plenum/persistence/req_id_to_txn.py @@ -1,6 +1,6 @@ import string -from collections import Iterable +from collections.abc import Iterable from common.exceptions import LogicError from storage.kv_store import KeyValueStorage diff --git a/plenum/server/batch_handlers/audit_batch_handler.py b/plenum/server/batch_handlers/audit_batch_handler.py index c2a207d2b4..4c5a249e44 100644 --- a/plenum/server/batch_handlers/audit_batch_handler.py +++ b/plenum/server/batch_handlers/audit_batch_handler.py @@ -1,4 +1,4 @@ -from collections import Iterable +from collections.abc import Iterable from common.exceptions import LogicError from ledger.ledger import Ledger diff --git a/plenum/server/consensus/checkpoint_service.py b/plenum/server/consensus/checkpoint_service.py index f5c2a3be2b..b99f05a77d 100644 --- a/plenum/server/consensus/checkpoint_service.py +++ b/plenum/server/consensus/checkpoint_service.py @@ -252,7 +252,7 @@ def _reset_checkpoints(self): # because according to paper, checkpoints cleared only when next stabilized. # Avoid using it while implement other services. self._data.checkpoints.clear() - self._data.checkpoints.append(self._data.initial_checkpoint) + self._data.checkpoints.add(self._data.initial_checkpoint) def __str__(self) -> str: return "{} - checkpoint_service".format(self._data.name) diff --git a/plenum/server/consensus/ordering_service.py b/plenum/server/consensus/ordering_service.py index fce071eda0..2eb349144a 100644 --- a/plenum/server/consensus/ordering_service.py +++ b/plenum/server/consensus/ordering_service.py @@ -1909,8 +1909,8 @@ def dequeue_pre_prepares(self): r = 0 while self.prePreparesPendingPrevPP and self._can_dequeue_pre_prepare( - *self.prePreparesPendingPrevPP.iloc[0]): - _, (pp, sender) = self.prePreparesPendingPrevPP.popitem(last=False) + *self.prePreparesPendingPrevPP.keys()[0]): + _, (pp, sender) = self.prePreparesPendingPrevPP.popitem(index=0) if not self._can_pp_seq_no_be_in_view(pp.viewNo, pp.ppSeqNo): self._discard(pp, "Pre-Prepare from a previous view", logger.debug) diff --git a/plenum/server/consensus/replica_service.py b/plenum/server/consensus/replica_service.py index 6ed1684f8e..4396a3dac0 100644 --- a/plenum/server/consensus/replica_service.py +++ b/plenum/server/consensus/replica_service.py @@ -98,7 +98,7 @@ def __init__(self, name: str, validators: List[str], view_no: int, self._add_ledgers() # TODO: This is just for testing purposes only - self._data.checkpoints.append( + self._data.checkpoints.add( Checkpoint(instId=0, viewNo=0, seqNoStart=0, seqNoEnd=0, digest='4F7BsTMVPKFshM1MwLf6y23cid6fL3xMpazVoF9krzUw')) diff --git a/plenum/server/replica_helper.py b/plenum/server/replica_helper.py index 923d5ad38b..a3b389850d 100644 --- a/plenum/server/replica_helper.py +++ b/plenum/server/replica_helper.py @@ -1,5 +1,5 @@ from _sha256 import sha256 -from collections import OrderedDict, defaultdict +from collections import defaultdict, OrderedDict from enum import IntEnum, unique from typing import List diff --git a/plenum/test/checkpoints/test_checkpoints_removal_after_catchup.py b/plenum/test/checkpoints/test_checkpoints_removal_after_catchup.py index 1afdffdfd1..4f94696479 100644 --- a/plenum/test/checkpoints/test_checkpoints_removal_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoints_removal_after_catchup.py @@ -26,7 +26,7 @@ def clear_checkpoints(txnPoolNodeSet): max_key=stable_cp_seq_no)) replica._consensus_data.checkpoints.clear() if len(own_stable_checkpoints) > 0: - replica._consensus_data.checkpoints.append(own_stable_checkpoints[0]) + replica._consensus_data.checkpoints.add(own_stable_checkpoints[0]) replica._checkpointer._received_checkpoints.clear() diff --git a/plenum/test/consensus/checkpoint_service/test_checkpoint_service_unit.py b/plenum/test/consensus/checkpoint_service/test_checkpoint_service_unit.py index 05b6f69bf1..09c4e570b4 100644 --- a/plenum/test/consensus/checkpoint_service/test_checkpoint_service_unit.py +++ b/plenum/test/consensus/checkpoint_service/test_checkpoint_service_unit.py @@ -121,7 +121,7 @@ def test_process_backup_catchup_msg(checkpoint_service, tconf, checkpoint): checkpoint_service._received_checkpoints[key] = {"frm"} checkpoint_service._received_checkpoints[new_key] = {"frm"} - checkpoint_service._data.checkpoints.append(checkpoint) + checkpoint_service._data.checkpoints.add(checkpoint) checkpoint_service._data.last_ordered_3pc = (checkpoint_service.view_no, till_seq_no) checkpoint_service.caught_up_till_3pc(checkpoint_service._data.last_ordered_3pc) diff --git a/plenum/test/consensus/view_change/test_view_change_service.py b/plenum/test/consensus/view_change/test_view_change_service.py index b59bf2dd0e..e8257cebc2 100644 --- a/plenum/test/consensus/view_change/test_view_change_service.py +++ b/plenum/test/consensus/view_change/test_view_change_service.py @@ -29,7 +29,7 @@ def _service(name): data.node_mode = Mode.participating digest = cp_digest(DEFAULT_STABLE_CHKP) cp = Checkpoint(instId=0, viewNo=initial_view_no, seqNoStart=0, seqNoEnd=DEFAULT_STABLE_CHKP, digest=digest) - data.checkpoints.append(cp) + data.checkpoints.add(cp) ViewChangeTriggerService(data=data, timer=timer, @@ -631,7 +631,7 @@ def test_do_not_send_instance_change_on_timeout_when_multiple_view_change_finish # receive quorum of ViewChanges and ViewChangeAcks non_primaries = [item for item in validators if item != primary_name] vc = create_view_change(initial_view_no + 1) - service._data.checkpoints.append(Checkpoint(instId=0, + service._data.checkpoints.add(Checkpoint(instId=0, viewNo=initial_view_no + 1, seqNoStart=0, seqNoEnd=DEFAULT_STABLE_CHKP, diff --git a/plenum/test/monitoring/test_stats_publisher.py b/plenum/test/monitoring/test_stats_publisher.py index ae8e6bd338..1c8a8f2502 100644 --- a/plenum/test/monitoring/test_stats_publisher.py +++ b/plenum/test/monitoring/test_stats_publisher.py @@ -17,8 +17,7 @@ def _acceptClient(clientReader, clientWriter): loop = asyncio.get_event_loop() server = loop.run_until_complete( asyncio.start_server(_acceptClient, - host=STATS_SERVER_IP, port=STATS_SERVER_PORT, - loop=loop)) + host=STATS_SERVER_IP, port=STATS_SERVER_PORT)) yield server server.close() loop.run_until_complete(server.wait_closed()) diff --git a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py index 7832d83b05..d9316151df 100644 --- a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py +++ b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py @@ -17,7 +17,7 @@ def _set_ppseqno(nodes, new_ppsn): repl._checkpointer.set_watermarks(low_watermark=new_ppsn) st_chk = new_ppsn - new_ppsn % 100 repl._consensus_data.stable_checkpoint = st_chk - repl._consensus_data.checkpoints.append(Checkpoint(0, repl.viewNo, st_chk - 100, st_chk, None)) + repl._consensus_data.checkpoints.add(Checkpoint(0, repl.viewNo, st_chk - 100, st_chk, None)) repl.last_ordered_3pc = (repl.viewNo, new_ppsn) diff --git a/setup.py b/setup.py index af9246306a..96aa05298d 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ with open(metadata['__file__'], 'r') as f: exec(f.read(), metadata) -tests_require = ['attrs==20.3.0', 'pytest==6.2.2', 'pytest-xdist==2.2.1', 'pytest-forked==1.3.0', +tests_require = ['attrs==20.3.0', 'pytest==6.2.5', 'pytest-xdist==2.2.1', 'pytest-forked==1.3.0', 'python3-indy==1.16.0.post236', 'pytest-asyncio==0.14.0'] @@ -126,14 +126,14 @@ def run(self): 'python-rocksdb', 'python-ursa==0.1.1', ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078745445) - 'rlp==0.6.0', + 'rlp==2.0.0', 'semver==2.13.0', # 'sha3==0.2.1', 'sha3', # 'six==1.15.0', 'six', ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118) - 'sortedcontainers==1.5.7', + 'sortedcontainers==2.1.0', ### Tests fail without version pin (GHA run: https://github.com/udosson/indy-plenum/actions/runs/1078741118) 'ujson==1.33', ], diff --git a/state/util/fast_rlp.py b/state/util/fast_rlp.py index 52e64443a5..fad84ae8c5 100644 --- a/state/util/fast_rlp.py +++ b/state/util/fast_rlp.py @@ -76,10 +76,13 @@ def consume_length_prefix(rlp, start): encode_optimized = _encode_optimized decode_optimized = _decode_optimized else: - encode_optimized = rlp.codec.encode_raw + # The raw encode doesn't support str + # encode_optimized = rlp.codec.encode_raw # rlp does not implement a decode_raw function. # decode_optimized = rlp.codec.decode_raw - decode_optimized = _decode_optimized + # decode_optimized = _decode_optimized + encode_optimized = rlp.encode + decode_optimized = rlp.decode def main():