(fix) Fix logging in shared eval file to prevent key disclosure (#4108) #2

Workflow file for this run

.github/workflows/ghcr-build.yml at c3bbe60

	# Workflow that builds, tests and then pushes the runtime docker images to the ghcr.io repository
	name: Build, Test and Publish RT Image

	# Only run one workflow of the same group at a time.
	# There can be at most one running and one pending job in a concurrency group at any time.
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

	# Always run on "main"
	# Always run on tags
	# Always run on PRs
	# Can also be triggered manually
	on:
	push:
	branches:
	- main
	tags:
	- '*'
	pull_request:
	workflow_dispatch:
	inputs:
	reason:
	description: 'Reason for manual trigger'
	required: true
	default: ''

	env:
	BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST: nikolaik/python-nodejs:python3.11-nodejs22

	jobs:
	# Builds the OpenHands Docker images
	ghcr_build_app:
	name: Build App Image
	runs-on: ubuntu-latest
	permissions:
	contents: read
	packages: write
	outputs:
	hash_from_app_image: ${{ steps.get_hash_in_app_image.outputs.hash_from_app_image }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/free-disk-space@main
	with:
	# this might remove tools that are actually needed,
	# if set to "true" but frees about 6 GB
	tool-cache: true
	# all of these default to true, but feel free to set to
	# "false" if necessary for your workflow
	android: true
	dotnet: true
	haskell: true
	large-packages: true
	docker-images: false
	swap-storage: true
	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3.0.0
	with:
	image: tonistiigi/binfmt:latest
	- name: Login to GHCR
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}
	- name: Set up Docker Buildx
	id: buildx
	uses: docker/setup-buildx-action@v3
	- name: Build and push app image
	if: "!github.event.pull_request.head.repo.fork"
	run: \|
	./containers/build.sh -i openhands -o ${{ github.repository_owner }} --push
	- name: Build app image
	if: "github.event.pull_request.head.repo.fork"
	run: \|
	./containers/build.sh -i openhands -o ${{ github.repository_owner }} --load
	- name: Get hash in App Image
	id: get_hash_in_app_image
	run: \|
	# Lowercase the repository owner
	export REPO_OWNER=${{ github.repository_owner }}
	REPO_OWNER=$(echo $REPO_OWNER \| tr '[:upper:]' '[:lower:]')
	# Run the build script in the app image
	docker run -e SANDBOX_USER_ID=0 -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/${REPO_OWNER}/openhands:${{ github.sha }} /bin/bash -c "mkdir -p containers/runtime; python3 openhands/runtime/utils/runtime_build.py --base_image ${{ env.BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST }} --build_folder containers/runtime --force_rebuild" 2>&1 \| tee docker-outputs.txt
	# Get the hash from the build script
	hash_from_app_image=$(cat docker-outputs.txt \| grep "Hash for docker build directory" \| awk -F "): " '{print $2}' \| uniq \| head -n1)
	echo "hash_from_app_image=$hash_from_app_image" >> $GITHUB_OUTPUT
	echo "Hash from app image: $hash_from_app_image"


	# Builds the runtime Docker images
	ghcr_build_runtime:
	name: Build Image
	runs-on: ubuntu-latest
	permissions:
	contents: read
	packages: write
	strategy:
	matrix:
	base_image:
	- image: 'nikolaik/python-nodejs:python3.11-nodejs22'
	tag: nikolaik
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/free-disk-space@main
	with:
	# this might remove tools that are actually needed,
	# if set to "true" but frees about 6 GB
	tool-cache: true
	# all of these default to true, but feel free to set to
	# "false" if necessary for your workflow
	android: true
	dotnet: true
	haskell: true
	large-packages: true
	docker-images: false
	swap-storage: true
	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3.0.0
	with:
	image: tonistiigi/binfmt:latest
	- name: Login to GHCR
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}
	- name: Set up Docker Buildx
	id: buildx
	uses: docker/setup-buildx-action@v3
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	- name: Cache Poetry dependencies
	uses: actions/cache@v4
	with:
	path: \|
	~/.cache/pypoetry
	~/.virtualenvs
	key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-
	- name: Install poetry via pipx
	run: pipx install poetry
	- name: Install Python dependencies using Poetry
	run: make install-python-dependencies
	- name: Create source distribution and Dockerfile
	run: poetry run python3 openhands/runtime/utils/runtime_build.py --base_image ${{ matrix.base_image.image }} --build_folder containers/runtime --force_rebuild
	- name: Build and push runtime image ${{ matrix.base_image.image }}
	if: github.event.pull_request.head.repo.fork != true
	run: \|
	./containers/build.sh -i runtime -o ${{ github.repository_owner }} --push -t ${{ matrix.base_image.tag }}
	# Forked repos can't push to GHCR, so we need to upload the image as an artifact
	- name: Build runtime image ${{ matrix.base_image.image }} for fork
	if: github.event.pull_request.head.repo.fork
	uses: docker/build-push-action@v6
	with:
	tags: ghcr.io/all-hands-ai/runtime:${{ github.sha }}-${{ matrix.base_image.tag }}
	outputs: type=docker,dest=/tmp/runtime-${{ matrix.base_image.tag }}.tar
	context: containers/runtime
	- name: Upload runtime image for fork
	if: github.event.pull_request.head.repo.fork
	uses: actions/upload-artifact@v4
	with:
	name: runtime-${{ matrix.base_image.tag }}
	path: /tmp/runtime-${{ matrix.base_image.tag }}.tar

	verify_hash_equivalence_in_runtime_and_app:
	name: Verify Hash Equivalence in Runtime and Docker images
	runs-on: ubuntu-latest
	needs: [ghcr_build_runtime, ghcr_build_app]
	strategy:
	fail-fast: false
	matrix:
	base_image: ['nikolaik']
	steps:
	- uses: actions/checkout@v4
	- name: Cache Poetry dependencies
	uses: actions/cache@v4
	with:
	path: \|
	~/.cache/pypoetry
	~/.virtualenvs
	key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	- name: Install poetry via pipx
	run: pipx install poetry
	- name: Install Python dependencies using Poetry
	run: make install-python-dependencies
	- name: Get hash in App Image
	run: \|
	echo "Hash from app image: ${{ needs.ghcr_build_app.outputs.hash_from_app_image }}"
	echo "hash_from_app_image=${{ needs.ghcr_build_app.outputs.hash_from_app_image }}" >> $GITHUB_ENV

	- name: Get hash using code (development mode)
	run: \|
	mkdir -p containers/runtime
	poetry run python3 openhands/runtime/utils/runtime_build.py --base_image ${{ env.BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST }} --build_folder containers/runtime --force_rebuild > output.txt 2>&1
	hash_from_code=$(cat output.txt \| grep "Hash for docker build directory" \| awk -F "): " '{print $2}' \| uniq \| head -n1)
	echo "hash_from_code=$hash_from_code" >> $GITHUB_ENV

	- name: Compare hashes
	run: \|
	echo "Hash from App Image: ${{ env.hash_from_app_image }}"
	echo "Hash from Code: ${{ env.hash_from_code }}"
	if [ "${{ env.hash_from_app_image }}" = "${{ env.hash_from_code }}" ]; then
	echo "Hashes match!"
	else
	echo "Hashes do not match!"
	exit 1
	fi

	# Run unit tests with the EventStream runtime Docker images as root
	test_runtime_root:
	name: RT Unit Tests (Root)
	needs: [ghcr_build_runtime]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	base_image: ['nikolaik']
	steps:
	- uses: actions/checkout@v4
	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/free-disk-space@main
	with:
	# this might remove tools that are actually needed,
	# if set to "true" but frees about 6 GB
	tool-cache: true
	# all of these default to true, but feel free to set to
	# "false" if necessary for your workflow
	android: true
	dotnet: true
	haskell: true
	large-packages: true
	docker-images: false
	swap-storage: true
	- name: Set up Docker Buildx
	id: buildx
	uses: docker/setup-buildx-action@v3
	# Forked repos can't push to GHCR, so we need to download the image as an artifact
	- name: Download runtime image for fork
	if: github.event.pull_request.head.repo.fork
	uses: actions/download-artifact@v4
	with:
	name: runtime-${{ matrix.base_image }}
	path: /tmp
	- name: Load runtime image for fork
	if: github.event.pull_request.head.repo.fork
	run: \|
	docker load --input /tmp/runtime-${{ matrix.base_image }}.tar
	- name: Cache Poetry dependencies
	uses: actions/cache@v4
	with:
	path: \|
	~/.cache/pypoetry
	~/.virtualenvs
	key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	- name: Install poetry via pipx
	run: pipx install poetry
	- name: Install Python dependencies using Poetry
	run: make install-python-dependencies
	- name: Run runtime tests
	run: \|
	# We install pytest-xdist in order to run tests across CPUs
	poetry run pip install pytest-xdist

	# Install to be able to retry on failures for flaky tests
	poetry run pip install pytest-rerunfailures

	image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ github.sha }}-${{ matrix.base_image }}
	image_name=$(echo $image_name \| tr '[:upper:]' '[:lower:]')

	SKIP_CONTAINER_LOGS=true \
	TEST_RUNTIME=eventstream \
	SANDBOX_USER_ID=$(id -u) \
	SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
	TEST_IN_CI=true \
	RUN_AS_OPENHANDS=false \
	poetry run pytest -n 3 -raR --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# Run unit tests with the EventStream runtime Docker images as openhands user
	test_runtime_oh:
	name: RT Unit Tests (openhands)
	runs-on: ubuntu-latest
	needs: [ghcr_build_runtime]
	strategy:
	matrix:
	base_image: ['nikolaik']
	steps:
	- uses: actions/checkout@v4
	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/free-disk-space@main
	with:
	# this might remove tools that are actually needed,
	# if set to "true" but frees about 6 GB
	tool-cache: true
	# all of these default to true, but feel free to set to
	# "false" if necessary for your workflow
	android: true
	dotnet: true
	haskell: true
	large-packages: true
	docker-images: false
	swap-storage: true
	- name: Set up Docker Buildx
	id: buildx
	uses: docker/setup-buildx-action@v3
	# Forked repos can't push to GHCR, so we need to download the image as an artifact
	- name: Download runtime image for fork
	if: github.event.pull_request.head.repo.fork
	uses: actions/download-artifact@v4
	with:
	name: runtime-${{ matrix.base_image }}
	path: /tmp
	- name: Load runtime image for fork
	if: github.event.pull_request.head.repo.fork
	run: \|
	docker load --input /tmp/runtime-${{ matrix.base_image }}.tar
	- name: Cache Poetry dependencies
	uses: actions/cache@v4
	with:
	path: \|
	~/.cache/pypoetry
	~/.virtualenvs
	key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	- name: Install poetry via pipx
	run: pipx install poetry
	- name: Install Python dependencies using Poetry
	run: make install-python-dependencies
	- name: Run runtime tests
	run: \|
	# We install pytest-xdist in order to run tests across CPUs
	poetry run pip install pytest-xdist

	# Install to be able to retry on failures for flaky tests
	poetry run pip install pytest-rerunfailures

	image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ github.sha }}-${{ matrix.base_image }}
	image_name=$(echo $image_name \| tr '[:upper:]' '[:lower:]')

	SKIP_CONTAINER_LOGS=true \
	TEST_RUNTIME=eventstream \
	SANDBOX_USER_ID=$(id -u) \
	SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
	TEST_IN_CI=true \
	RUN_AS_OPENHANDS=true \
	poetry run pytest -n 3 -raR --reruns 1 --reruns-delay 3 --cov=agenthub --cov=openhands --cov-report=xml -s ./tests/runtime
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# Run integration tests with the eventstream runtime Docker image
	runtime_integration_tests_on_linux:
	name: RT Integration Tests (Linux)
	runs-on: ubuntu-latest
	needs: [ghcr_build_runtime]
	strategy:
	fail-fast: false
	matrix:
	base_image: ['nikolaik']
	steps:
	- uses: actions/checkout@v4
	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/free-disk-space@main
	with:
	# this might remove tools that are actually needed,
	# if set to "true" but frees about 6 GB
	tool-cache: true
	# all of these default to true, but feel free to set to
	# "false" if necessary for your workflow
	android: true
	dotnet: true
	haskell: true
	large-packages: true
	docker-images: false
	swap-storage: true
	- name: Set up Docker Buildx
	id: buildx
	uses: docker/setup-buildx-action@v3
	# Forked repos can't push to GHCR, so we need to download the image as an artifact
	- name: Download runtime image for fork
	if: github.event.pull_request.head.repo.fork
	uses: actions/download-artifact@v4
	with:
	name: runtime-${{ matrix.base_image }}
	path: /tmp
	- name: Load runtime image for fork
	if: github.event.pull_request.head.repo.fork
	run: \|
	docker load --input /tmp/runtime-${{ matrix.base_image }}.tar
	- name: Cache Poetry dependencies
	uses: actions/cache@v4
	with:
	path: \|
	~/.cache/pypoetry
	~/.virtualenvs
	key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
	restore-keys: \|
	${{ runner.os }}-poetry-
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.11'
	- name: Install poetry via pipx
	run: pipx install poetry
	- name: Install Python dependencies using Poetry
	run: make install-python-dependencies
	- name: Run integration tests
	run: \|
	image_name=ghcr.io/${{ github.repository_owner }}/runtime:${{ github.sha }}-${{ matrix.base_image }}
	image_name=$(echo $image_name \| tr '[:upper:]' '[:lower:]')

	TEST_RUNTIME=eventstream \
	SANDBOX_USER_ID=$(id -u) \
	SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
	TEST_IN_CI=true \
	TEST_ONLY=true \
	./tests/integration/regenerate.sh
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# The two following jobs (named identically) are to check whether all the runtime tests have passed as the
	# "All Runtime Tests Passed" is a required job for PRs to merge
	# Due to this bug: https://github.com/actions/runner/issues/2566, we want to create a job that runs when the
	# prerequisites have been cancelled or failed so merging is disallowed, otherwise Github considers "skipped" as "success"
	runtime_tests_check_success:
	name: All Runtime Tests Passed
	if: ${{ !cancelled() && !contains(needs..result, 'failure') && !contains(needs..result, 'cancelled') }}
	runs-on: ubuntu-latest
	needs: [test_runtime_root, test_runtime_oh, runtime_integration_tests_on_linux, verify_hash_equivalence_in_runtime_and_app]
	steps:
	- name: All tests passed
	run: echo "All runtime tests have passed successfully!"

	runtime_tests_check_fail:
	name: All Runtime Tests Passed
	if: ${{ cancelled() \|\| contains(needs..result, 'failure') \|\| contains(needs..result, 'cancelled') }}
	runs-on: ubuntu-latest
	needs: [test_runtime_root, test_runtime_oh, runtime_integration_tests_on_linux, verify_hash_equivalence_in_runtime_and_app]
	steps:
	- name: Some tests failed
	run: \|
	echo "Some runtime tests failed or were cancelled"
	exit 1

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

(fix) Fix logging in shared eval file to prevent key disclosure (#4108) #2

Workflow file

(fix) Fix logging in shared eval file to prevent key disclosure (#4108) #2

Jobs

Run details

Workflow file for this run