diff --git a/.github/workflows/generate_workflows.py b/.github/workflows/generate_workflows.py index ea46533409..5413019528 100644 --- a/.github/workflows/generate_workflows.py +++ b/.github/workflows/generate_workflows.py @@ -1,19 +1,205 @@ +from collections import defaultdict from pathlib import Path +from re import compile as re_compile -from generate_workflows_lib import ( - generate_lint_workflow, - generate_misc_workflow, - generate_test_workflow, +from jinja2 import Environment, FileSystemLoader +from tox.config.cli.parse import get_options +from tox.config.sets import CoreConfigSet +from tox.config.source.tox_ini import ToxIni +from tox.session.state import State + +_tox_test_env_regex = re_compile( + r"(?Ppy\w+)-test-" + r"(?P[-\w]+\w)-?(?P\d+)?" +) +_tox_lint_env_regex = re_compile(r"lint-(?P[-\w]+)") +_tox_contrib_env_regex = re_compile( + r"py38-test-(?P[-\w]+\w)-?(?P\d+)?" ) -tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini") -workflows_directory_path = Path(__file__).parent -generate_test_workflow( - tox_ini_path, - workflows_directory_path, - "ubuntu-latest", - "windows-latest", -) -generate_lint_workflow(tox_ini_path, workflows_directory_path) -generate_misc_workflow(tox_ini_path, workflows_directory_path) +def get_tox_envs(tox_ini_path: Path) -> list: + tox_ini = ToxIni(tox_ini_path) + + conf = State(get_options(), []).conf + + tox_section = next(tox_ini.sections()) + + core_config_set = CoreConfigSet( + conf, tox_section, tox_ini_path.parent, tox_ini_path + ) + + ( + core_config_set.loaders.extend( + tox_ini.get_loaders( + tox_section, + base=[], + override_map=defaultdict(list, {}), + conf=core_config_set, + ) + ) + ) + + return core_config_set.load("env_list") + + +def get_test_job_datas(tox_envs: list, operating_systems: list) -> list: + os_alias = {"ubuntu-latest": "Ubuntu", "windows-latest": "Windows"} + + python_version_alias = { + "pypy3": "pypy-3.8", + "py38": "3.8", + "py39": "3.9", + "py310": "3.10", + "py311": "3.11", + "py312": "3.12", + } + + test_job_datas = [] + + for operating_system in operating_systems: + for tox_env in tox_envs: + tox_test_env_match = _tox_test_env_regex.match(tox_env) + + if tox_test_env_match is None: + continue + + groups = tox_test_env_match.groupdict() + + aliased_python_version = python_version_alias[ + groups["python_version"] + ] + tox_env = tox_test_env_match.string + + test_requirements = groups["test_requirements"] + + if test_requirements is None: + test_requirements = " " + + else: + test_requirements = f"-{test_requirements} " + + test_job_datas.append( + { + "name": f"{tox_env}_{operating_system}", + "ui_name": ( + f"{groups['name']}" + f"{test_requirements}" + f"{aliased_python_version} " + f"{os_alias[operating_system]}" + ), + "python_version": aliased_python_version, + "tox_env": tox_env, + "os": operating_system, + } + ) + + return test_job_datas + + +def get_lint_job_datas(tox_envs: list) -> list: + lint_job_datas = [] + + for tox_env in tox_envs: + tox_lint_env_match = _tox_lint_env_regex.match(tox_env) + + if tox_lint_env_match is None: + continue + + tox_env = tox_lint_env_match.string + + lint_job_datas.append( + { + "name": f"{tox_env}", + "ui_name": f"{tox_lint_env_match.groupdict()['name']}", + "tox_env": tox_env, + } + ) + + return lint_job_datas + + +def get_misc_job_datas(tox_envs: list) -> list: + regex_patterns = [ + _tox_test_env_regex, + _tox_lint_env_regex, + _tox_contrib_env_regex, + re_compile(r"benchmark.+"), + ] + + return [ + tox_env + for tox_env in tox_envs + if not any(pattern.match(tox_env) for pattern in regex_patterns) + ] + + +def _generate_workflow( + job_datas: list, + template_name: str, + output_dir: Path, + max_jobs: int = 250, +): + # Github seems to limit the amount of jobs in a workflow file, that is why + # they are split in groups of 250 per workflow file. + for file_number, job_datas in enumerate( + [ + job_datas[index : index + max_jobs] + for index in range(0, len(job_datas), max_jobs) + ] + ): + with open( + output_dir.joinpath(f"{template_name}_{file_number}.yml"), "w" + ) as test_yml_file: + test_yml_file.write( + Environment( + loader=FileSystemLoader( + Path(__file__).parent.joinpath("templates") + ) + ) + .get_template(f"{template_name}.yml.j2") + .render(job_datas=job_datas, file_number=file_number) + ) + test_yml_file.write("\n") + + +def generate_test_workflow( + tox_ini_path: Path, workflow_directory_path: Path, operating_systems +) -> None: + _generate_workflow( + get_test_job_datas(get_tox_envs(tox_ini_path), operating_systems), + "test", + workflow_directory_path, + ) + + +def generate_lint_workflow( + tox_ini_path: Path, + workflow_directory_path: Path, +) -> None: + _generate_workflow( + get_lint_job_datas(get_tox_envs(tox_ini_path)), + "lint", + workflow_directory_path, + ) + + +def generate_misc_workflow( + tox_ini_path: Path, + workflow_directory_path: Path, +) -> None: + _generate_workflow( + get_misc_job_datas(get_tox_envs(tox_ini_path)), + "misc", + workflow_directory_path, + ) + + +if __name__ == "__main__": + tox_ini_path = Path(__file__).parent.parent.parent.joinpath("tox.ini") + output_dir = Path(__file__).parent + generate_test_workflow( + tox_ini_path, output_dir, ["ubuntu-latest", "windows-latest"] + ) + generate_lint_workflow(tox_ini_path, output_dir) + generate_misc_workflow(tox_ini_path, output_dir) diff --git a/.github/workflows/misc_0.yml b/.github/workflows/misc_0.yml index 0ce4663129..c1ea218fd6 100644 --- a/.github/workflows/misc_0.yml +++ b/.github/workflows/misc_0.yml @@ -233,7 +233,7 @@ jobs: - name: Run tests run: tox -e generate-workflows - - name: Check workflows are up to date + - name: Check github workflows are up to date run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1) ruff: diff --git a/.github/workflows/templates/lint.yml.j2 b/.github/workflows/templates/lint.yml.j2 new file mode 100644 index 0000000000..6959261bba --- /dev/null +++ b/.github/workflows/templates/lint.yml.j2 @@ -0,0 +1,37 @@ +# Do not edit this file. +# This file is generated automatically by executing tox -e generate-workflows + +name: Lint {{ file_number }} + +on: + push: + branches-ignore: + - 'release/*' + pull_request: + +env: + CORE_REPO_SHA: main + CONTRIB_REPO_SHA: main + PIP_EXISTS_ACTION: w + +jobs: + {%- for job_data in job_datas %} + + {{ job_data.name }}: + name: {{ job_data.ui_name }} + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e {{ job_data.tox_env }} + {%- endfor %} diff --git a/.github/workflows/templates/misc.yml.j2 b/.github/workflows/templates/misc.yml.j2 new file mode 100644 index 0000000000..928d06c04e --- /dev/null +++ b/.github/workflows/templates/misc.yml.j2 @@ -0,0 +1,69 @@ +# Do not edit this file. +# This file is generated automatically by executing tox -e generate-workflows + +name: Misc {{ file_number }} + +on: + push: + branches-ignore: + - 'release/*' + pull_request: + +env: + CORE_REPO_SHA: main + CONTRIB_REPO_SHA: main + PIP_EXISTS_ACTION: w + +jobs: + {%- for job_data in job_datas %} + + {{ job_data }}: + name: {{ job_data }} + runs-on: ubuntu-latest + {%- if job_data == "generate-workflows" %} + if: | + !contains(github.event.pull_request.labels.*.name, 'Skip generate-workflows') + && github.event.pull_request.user.login != 'opentelemetrybot' && github.event_name == 'pull_request' + {%- endif %} + {%- if job_data == "public-symbols-check" %} + if: | + !contains(github.event.pull_request.labels.*.name, 'Approve Public API check') + && github.actor != 'opentelemetrybot' && github.event_name == 'pull_request' + {%- endif %} + {%- if job_data == "docs" %} + if: | + github.event.pull_request.user.login != 'opentelemetrybot' && github.event_name == 'pull_request' + {%- endif %} + steps: + - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %} + uses: actions/checkout@v4 + {%- if job_data == "public-symbols-check" %} + with: + fetch-depth: 0 + + - name: Checkout main + run: git checkout main + + - name: Pull origin + run: git pull --rebase=false origin main + + - name: Checkout pull request + run: git checkout ${% raw %}{{ github.event.pull_request.head.sha }}{% endraw %} + {%- endif %} + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e {{ job_data }} + {%- if job_data == "generate-workflows" %} + + - name: Check github workflows are up to date + run: git diff --exit-code || (echo 'Generated workflows are out of date, run "tox -e generate-workflows" and commit the changes in this PR.' && exit 1) + {%- endif %} + {%- endfor %} diff --git a/.github/workflows/templates/test.yml.j2 b/.github/workflows/templates/test.yml.j2 new file mode 100644 index 0000000000..e5168470d8 --- /dev/null +++ b/.github/workflows/templates/test.yml.j2 @@ -0,0 +1,42 @@ +# Do not edit this file. +# This file is generated automatically by executing tox -e generate-workflows + +name: Test {{ file_number }} + +on: + push: + branches-ignore: + - 'release/*' + pull_request: + +env: + CORE_REPO_SHA: main + CONTRIB_REPO_SHA: main + PIP_EXISTS_ACTION: w + +jobs: + {%- for job_data in job_datas %} + + {{ job_data.name }}: + name: {{ job_data.ui_name }} + runs-on: {{ job_data.os }} + steps: + - name: Checkout repo @ SHA - ${% raw %}{{ github.sha }}{% endraw %} + uses: actions/checkout@v4 + + - name: Set up Python {{ job_data.python_version }} + uses: actions/setup-python@v5 + with: + python-version: "{{ job_data.python_version }}" + + - name: Install tox + run: pip install tox + {%- if job_data.os == "windows-latest" %} + + - name: Configure git to support long filenames + run: git config --system core.longpaths true + {%- endif %} + + - name: Run tests + run: tox -e {{ job_data.tox_env }} -- -ra + {%- endfor %} diff --git a/tox.ini b/tox.ini index 3e92195f9d..f9a4c8e091 100644 --- a/tox.ini +++ b/tox.ini @@ -334,9 +334,9 @@ commands = python {toxinidir}/scripts/public_symbols_checker.py [testenv:generate-workflows] -recreate = True deps = - {env:CONTRIB_REPO}\#egg=generate_workflows_lib&subdirectory=.github/workflows/generate_workflows_lib + tox + Jinja2 commands = python {toxinidir}/.github/workflows/generate_workflows.py