Add release scripts and workflow validate-repackaged-size (#6059)

Move release scripts and workflow to validate-repackaged-size of
binaries to test Infra

.github/workflows/validate-repackaged-binary-sizes.yml

@@ -0,0 +1,92 @@
+name: Validate manywheel binaries
+
+# This workflow validates the size of the manywheel binaries after repackaging for PyPi
+# Specify the direct URLs to the binaries (from https://download.pytorch.org/whl/test/torch/) in the matrix
+# along with the python version.
+#
+# The workflow will:
+#  * download the binaries,
+#  * run release/pypi/prep_binary_for_pypi.sh
+#  * run smoke tests on the repackaged binaries
+#  * display the size before and after repackaging as the workflow annotation
+#  * optionally upload the repackaged binaries as artifacts (for debug or promotion)
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/validate-repackaged-binary-sizes.yml
+      - release/pypi/prep_binary_for_pypi.sh
+
+jobs:
+  generate-linux-matrix:
+    uses: ./.github/workflows/generate_binary_build_matrix.yml
+    with:
+      package-type: wheel
+      os: linux
+      channel: test
+      with-xpu: disable
+      with-rocm: disable
+
+  validate-binary-size:
+    needs: generate-linux-matrix
+    strategy:
+      matrix: ${{ fromJson(needs.generate-linux-matrix.outputs.matrix) }}
+      fail-fast: false
+    uses: ./.github/workflows/linux_job.yml
+    name: ${{ matrix.build_name }}
+    with:
+      runner: ${{ matrix.validation_runner }}
+      job-name: "Validate binary size"
+      repository: "pytorch/pytorch"
+      docker-build-dir: "skip-docker-build"
+      binary-matrix: ${{ toJSON(matrix) }}
+      ref: main
+      script: |
+        set -ex
+
+        # skip testing on cu126
+        if [[ ${MATRIX_DESIRED_CUDA} == "cu126" ]]; then
+          exit 0
+        fi
+
+        whl_suffix="linux_x86_64.whl"
+        pyndt="$(echo $MATRIX_PYTHON_VERSION | tr -d m.u)"
+        base_url="https://download.pytorch.org/whl/${MATRIX_CHANNEL}/${MATRIX_DESIRED_CUDA}"
+        whl_url="${base_url}/torch-${MATRIX_STABLE_VERSION}%2B${MATRIX_DESIRED_CUDA}-cp${pyndt}-cp${pyndt}-${whl_suffix}"
+
+        export ENV_NAME="conda-env-${{ github.run_id }}"
+        export ENV_NAME="conda-env-${{ github.run_id }}"
+        export TARGET_OS="linux"
+
+        # install zip
+        sudo yum install zip -y
+
+        # install patchelf
+        chmod a+x ./.ci/docker/common/install_patchelf.sh
+        sudo ./.ci/docker/common/install_patchelf.sh
+
+        # download torch whl
+        wget ${whl_url}
+        FILENAME=$(ls -1 *.whl | head -n 1)
+        SIZE_BEFORE=$(du -h $FILENAME | cut -f1)
+
+        # repackage into manywheel
+        ../../test-infra/release/pypi/prep_binary_for_pypi.sh $FILENAME
+
+        NEW_FILENAME=$(ls -1 *.whl | head -n 1)
+        echo "::notice:: $FILENAME before: $SIZE_BEFORE after: $(du -h $NEW_FILENAME | cut -f1)"
+
+        # cp to ${RUNNER_ARTIFACT_DIR}
+        cp $NEW_FILENAME ${RUNNER_ARTIFACT_DIR}/
+
+        # create conda env
+        conda create -y -n $ENV_NAME python=${MATRIX_PYTHON_VERSION}
+        conda activate $ENV_NAME
+
+        # install torch
+        pip install numpy pillow $NEW_FILENAME --extra-index-url ${base_url}
+
+        pushd ./.ci/pytorch/smoke_test
+        # run smoke test
+        python smoke_test.py --package=torchonly --torch-compile-check disabled
+        popd

release/README.md

@@ -0,0 +1,49 @@
+# PyTorch Release Scripts
+
+These are a collection of scripts that are to be used for release activities.
+
+> NOTE: All scripts should do no actual work unless the `DRY_RUN` environment variable is set
+>       to `disabled`.
+>       The basic idea being that there should be no potential to do anything dangerous unless
+>       `DRY_RUN` is explicitly set to `disabled`.
+
+## Requirements to actually run these scripts
+* AWS access to pytorch account (META Employees: `bunnylol cloud pytorch`)
+* Access to upload conda packages to the [`pytorch`](https://anaconda.org/pytorch) conda channel
+* Access to the PyPI repositories (like [torch](https://pypi.org/project/torch))
+
+## Promote pypi to staging
+
+Following steps needed in order to promote pypi to staging:
+1. Edit `release_versions.sh` and set correct version
+2. Run promote script : `./pypi/promote_pypi_to_staging.sh`
+3. Edit and run `../analytics/validate_pypi_staging.py` to perform initial prevalidation of binaries for pypi promotion
+4. Manually inspect and spot check binaries staged for pypi promotion by logging into s3 and downloading packages
+
+## Promote
+
+These are scripts related to promotion of release candidates to GA channels, these
+can actually be used to promote pytorch, libtorch, and related domain libraries.
+
+> NOTE: Currently the script requires some knowledge on when to comment things out / comment things in
+
+> TODO: Make the script not rely on commenting things out / commenting this in
+
+### Usage
+
+```bash
+./promote.sh
+```
+
+## Restoring backups
+
+All release candidates from `pytorch/pytorch` are currently backed up
+to `s3://pytorch-backup/${TAG_NAME}` and can be restored to the test channels with the
+`restore-backup.sh` script.
+
+Which backup to restore from is dictated by the `RESTORE_FROM` environment variable.
+
+### Usage
+```bash
+RESTORE_FROM=v1.5.0-rc5 ./restore-backup.sh
+```

release/promote.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "${DIR}/release_versions.sh"
+
+# Make sure to update these versions when doing a release first
+PYTORCH_VERSION=${PYTORCH_VERSION:-2.3.0}
+TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.18.0}
+TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.3.0}
+TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.18.0}
+TORCHREC_VERSION=${TORCHREC_VERSION:-0.8.0}
+TENSORRT_VERSION=${TENSORRT_VERSION:-2.2.0}
+FBGEMMGPU_VERSION=${FBGEMMGPU_VERSION:-1.0.0}
+
+DRY_RUN=${DRY_RUN:-enabled}
+
+promote_s3() {
+    local package_name
+    package_name=$1
+    local package_type
+    package_type=$2
+    local promote_version
+    promote_version=$3
+
+    echo "=-=-=-= Promoting ${package_name}'s v${promote_version} ${package_type} packages' =-=-=-="
+    (
+        set -x
+        TEST_PYTORCH_PROMOTE_VERSION="${promote_version}" \
+            PACKAGE_NAME="${package_name}" \
+            PACKAGE_TYPE="${package_type}" \
+            TEST_WITHOUT_GIT_TAG=1 \
+            DRY_RUN="${DRY_RUN}" ${DIR}/promote/s3_to_s3.sh
+    )
+    echo
+}
+
+promote_conda() {
+    local package_name
+    package_name=$1
+    local package_type
+    package_type=$2
+    local promote_version
+    promote_version=$3
+    echo "=-=-=-= Promoting ${package_name}'s v${promote_version} ${package_type} packages' =-=-=-="
+    (
+        ANACONDA="echo + anaconda"
+        if [[ "${DRY_RUN:-enabled}" = "disabled" ]]; then
+            ANACONDA="anaconda"
+            set -x
+        else
+            echo "DRY_RUN enabled not actually doing work"
+        fi
+        ${ANACONDA} copy --to-owner ${PYTORCH_CONDA_TO:-pytorch} ${PYTORCH_CONDA_FROM:-pytorch-test}/${package_name}/${promote_version}
+    )
+    echo
+}
+
+promote_pypi() {
+    local package_name
+    package_name=$1
+    local promote_version
+    promote_version=$2
+    echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi' =-=-=-="
+    (
+        set -x
+        TEST_PYTORCH_PROMOTE_VERSION="${promote_version}" \
+            PACKAGE_NAME="${package_name}" \
+            TEST_WITHOUT_GIT_TAG=1 \
+            DRY_RUN="${DRY_RUN}" ${DIR}/promote/wheel_to_pypi.sh
+    )
+    echo
+}
+
+# Promote s3 dependencies
+# promote_s3 "certifi" whl "2022.12.7"
+# promote_s3 "charset_normalizer" whl "2.1.1"
+# promote_s3 "cmake" whl "3.25"
+# promote_s3 "colorama" whl "0.4.6"
+# promote_s3 "triton" whl "2.0.0"
+# promote_s3 "pytorch_triton_rocm" whl "2.0.1"
+# promote_s3 "tqdm" whl "4.64.1"
+# promote_s3 "Pillow" whl "9.3.0"
+# for python 3.8-3.11
+# promote_s3 "numpy" whl "1.24.1"
+# for python 3.7 older pytorch versions
+# promote_s3 "numpy" whl "1.21.6"
+# promote_s3 "urllib3" whl "1.26.13"
+# promote_s3 "lit" whl "15.0.7"
+# promote_s3 "sympy" whl "1.11.1"
+# promote_s3 "typing_extensions" whl "4.4.0"
+# promote_s3 "filelock" whl "3.9.0"
+# promote_s3 "mpmath" whl "1.2.1"
+# promote_s3 "MarkupSafe" whl "2.1.2"
+# promote_s3 "Jinja2" whl "3.1.2"
+# promote_s3 "idna" whl "3.4"
+# promote_s3 "networkx" whl "3.0"
+# promote_s3 "packaging" whl "22.0"
+# promote_s3 "requests" whl "2.28.1"
+
+# promote_s3 torch whl "${PYTORCH_VERSION}"
+# promote_s3 torchvision whl "${TORCHVISION_VERSION}"
+# promote_s3 torchaudio whl "${TORCHAUDIO_VERSION}"
+# promote_s3 torchtext whl "${TORCHTEXT_VERSION}"
+# promote_s3 torchrec whl "${TORCHREC_VERSION}"
+# promote_s3 fbgemm-gpu whl "${FBGEMMGPU_VERSION}"
+# promote_s3 "libtorch-*" libtorch "${PYTORCH_VERSION}"
+# promote_s3 "torch_tensorrt" whl "${TENSORRT_VERSION}"
+
+# promote_conda torchtriton conda "2.1.0"
+# promote_conda pytorch-cuda conda "11.8"
+# promote_conda pytorch-cuda conda "12.1"
+
+# promote_conda pytorch conda "${PYTORCH_VERSION}"
+# promote_conda torchvision conda "${TORCHVISION_VERSION}"
+# promote_conda torchaudio conda "${TORCHAUDIO_VERSION}"
+# promote_conda torchtext conda "${TORCHTEXT_VERSION}"
+
+# Uncomment these to promote to pypi
+LINUX_VERSION_SUFFIX="%2Bcu102"
+WIN_VERSION_SUFFIX="%2Bcpu"
+# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torch "${PYTORCH_VERSION}"
+# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torch "${PYTORCH_VERSION}"
+# PLATFORM="win_amd64"    VERSION_SUFFIX="${WIN_VERSION_SUFFIX}"   promote_pypi torch "${PYTORCH_VERSION}"
+# PLATFORM="macosx_11_0"  VERSION_SUFFIX=""                        promote_pypi torch "${PYTORCH_VERSION}" # m1 mac
+
+# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torchvision "${TORCHVISION_VERSION}"
+# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torchvision "${TORCHVISION_VERSION}"
+# PLATFORM="win_amd64"    VERSION_SUFFIX="${WIN_VERSION_SUFFIX}"   promote_pypi torchvision "${TORCHVISION_VERSION}"
+# PLATFORM="macosx_11_0"  VERSION_SUFFIX=""                        promote_pypi torchvision "${TORCHVISION_VERSION}"
+
+# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torchaudio "${TORCHAUDIO_VERSION}"
+# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torchaudio "${TORCHAUDIO_VERSION}"
+# PLATFORM="win_amd64"    VERSION_SUFFIX="${WIN_VERSION_SUFFIX}"   promote_pypi torchaudio "${TORCHAUDIO_VERSION}"
+# PLATFORM="macosx_11_0"  VERSION_SUFFIX=""                        promote_pypi torchaudio "${TORCHAUDIO_VERSION}"

release/promote/common_utils.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+exit_if_not_on_git_tag() {
+    # Have an override for debugging purposes
+    if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]] ;then
+        >&2 echo "+ WARN: Continuing without being on a git tag"
+        exit 0
+    fi
+    # Exit if we're not currently on a git tag
+    if ! git describe --tags --exact >/dev/null 2>/dev/null; then
+        >&2 echo "- ERROR: Attempting to promote on a non-git tag, must have tagged current commit locally first"
+        exit 1
+    fi
+    # Exit if we're currently on an RC
+    if git describe --tags | grep "-rc" >/dev/null 2>/dev/null; then
+        >&2 echo "- ERROR: Attempting to promote on a non GA git tag, current tag must be a GA tag"
+        >&2 echo "         Example: v1.5.0"
+        exit 1
+    fi
+}
+
+get_pytorch_version() {
+    if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]];then
+        if  [[ -z "${TEST_PYTORCH_PROMOTE_VERSION-}" ]]; then
+            >&2 echo "- ERROR: Specified TEST_WITHOUT_GIT_TAG without specifying TEST_PYTORCH_PROMOTE_VERSION"
+            >&2 echo "-        TEST_PYTORCH_PROMOTE_VERSION must be specified"
+            exit 1
+        else
+            echo "${TEST_PYTORCH_PROMOTE_VERSION}"
+            exit 0
+        fi
+    fi
+    exit_if_not_on_git_tag
+    # Echo git tag, strip leading v
+    git describe --tags | sed -e 's/^v//'
+}
+
+aws_promote() {
+    package_name=$1
+    pytorch_version=$(get_pytorch_version)
+    # Dry run by default
+    DRY_RUN=${DRY_RUN:-enabled}
+    DRY_RUN_FLAG="--dryrun"
+    if [[ $DRY_RUN = "disabled" ]]; then
+        DRY_RUN_FLAG=""
+    fi
+    AWS=${AWS:-aws}
+    (
+        set -x
+        ${AWS} s3 cp ${DRY_RUN_FLAG} \
+            --only-show-errors \
+            --acl public-read \
+            --recursive \
+            --exclude '*' \
+            --include "*${package_name}-${pytorch_version}*" \
+            "${PYTORCH_S3_FROM/\/$//}" \
+            "${PYTORCH_S3_TO/\/$//}"
+    )
+    # ^ We grep for package_name-.*pytorch_version to avoid any situations where domain libraries have
+    #   the same version on our S3 buckets
+}