-
Notifications
You must be signed in to change notification settings - Fork 223
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Revert "Cleanup docker image build workflows and scripts" (#1934)
This reverts commit 68d5a48.
- Loading branch information
Showing
51 changed files
with
4,954 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
name: Build conda docker images | ||
|
||
on: | ||
push: | ||
branches: | ||
- main | ||
- release/* | ||
tags: | ||
# NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds | ||
# Release candidate tags look like: v1.11.0-rc1 | ||
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ | ||
paths: | ||
- conda/Dockerfile | ||
- 'common/*' | ||
- .github/workflows/build-conda-images.yml | ||
pull_request: | ||
paths: | ||
- conda/Dockerfile | ||
- 'common/*' | ||
- .github/workflows/build-conda-images.yml | ||
|
||
env: | ||
DOCKER_REGISTRY: "docker.io" | ||
DOCKER_BUILDKIT: 1 | ||
DOCKER_ID: ${{ secrets.DOCKER_ID }} | ||
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | ||
WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} | ||
|
||
jobs: | ||
build-docker: | ||
runs-on: linux.12xlarge.ephemeral | ||
strategy: | ||
matrix: | ||
cuda_version: ["11.8", "12.1", "12.4", "cpu"] | ||
env: | ||
CUDA_VERSION: ${{ matrix.cuda_version }} | ||
steps: | ||
- name: Checkout PyTorch builder | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
conda/build_docker.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
name: Build libtorch docker images | ||
|
||
on: | ||
push: | ||
branches: | ||
- main | ||
- release/* | ||
tags: | ||
# NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds | ||
# Release candidate tags look like: v1.11.0-rc1 | ||
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ | ||
paths: | ||
- .github/workflows/build-libtorch-images.yml | ||
- libtorch/Dockerfile | ||
- libtorch/build_docker.sh | ||
- 'common/*' | ||
pull_request: | ||
paths: | ||
- .github/workflows/build-libtorch-images.yml | ||
- libtorch/Dockerfile | ||
- 'common/*' | ||
- libtorch/build_docker.sh | ||
|
||
env: | ||
DOCKER_REGISTRY: "docker.io" | ||
DOCKER_BUILDKIT: 1 | ||
DOCKER_ID: ${{ secrets.DOCKER_ID }} | ||
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | ||
WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} | ||
|
||
jobs: | ||
build-docker-cuda: | ||
runs-on: linux.12xlarge.ephemeral | ||
strategy: | ||
matrix: | ||
cuda_version: ["12.4", "12.1", "11.8"] | ||
env: | ||
GPU_ARCH_TYPE: cuda | ||
GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | ||
steps: | ||
- name: Checkout PyTorch builder | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
libtorch/build_docker.sh | ||
build-docker-rocm: | ||
runs-on: linux.12xlarge.ephemeral | ||
strategy: | ||
matrix: | ||
rocm_version: ["6.0", "6.1"] | ||
env: | ||
GPU_ARCH_TYPE: rocm | ||
GPU_ARCH_VERSION: ${{ matrix.rocm_version }} | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
libtorch/build_docker.sh | ||
build-docker-cpu: | ||
runs-on: ubuntu-22.04 | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
libtorch/build_docker.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,210 @@ | ||
name: Build manywheel docker images | ||
|
||
on: | ||
push: | ||
branches: | ||
- main | ||
- release/* | ||
tags: | ||
# NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds | ||
# Release candidate tags look like: v1.11.0-rc1 | ||
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ | ||
paths: | ||
- .github/workflows/build-manywheel-images.yml | ||
- manywheel/Dockerfile | ||
- manywheel/Dockerfile_2_28 | ||
- manywheel/Dockerfile_aarch64 | ||
- manywheel/Dockerfile_2_28_aarch64 | ||
- manywheel/Dockerfile_cuda_aarch64 | ||
- manywheel/Dockerfile_cxx11-abi | ||
- manywheel/build_docker.sh | ||
- 'common/*' | ||
pull_request: | ||
paths: | ||
- .github/workflows/build-manywheel-images.yml | ||
- manywheel/Dockerfile | ||
- manywheel/Dockerfile_2_28 | ||
- manywheel/Dockerfile_aarch64 | ||
- manywheel/Dockerfile_2_28_aarch64 | ||
- manywheel/Dockerfile_cuda_aarch64 | ||
- manywheel/Dockerfile_cxx11-abi | ||
- 'common/*' | ||
- manywheel/build_docker.sh | ||
|
||
env: | ||
DOCKER_REGISTRY: "docker.io" | ||
DOCKER_BUILDKIT: 1 | ||
DOCKER_ID: ${{ secrets.DOCKER_ID }} | ||
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | ||
WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} | ||
|
||
jobs: | ||
build-docker-cuda: | ||
runs-on: linux.12xlarge.ephemeral | ||
strategy: | ||
matrix: | ||
cuda_version: ["12.4", "12.1", "11.8"] | ||
env: | ||
GPU_ARCH_TYPE: cuda | ||
GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | ||
steps: | ||
- name: Purge tools folder (free space for build) | ||
run: rm -rf /opt/hostedtoolcache | ||
- name: Checkout PyTorch builder | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
# NOTE: manylinux_2_28 are still experimental, see https://github.com/pytorch/pytorch/issues/123649 | ||
build-docker-cuda-manylinux_2_28: | ||
runs-on: linux.12xlarge.ephemeral | ||
strategy: | ||
matrix: | ||
cuda_version: ["12.4", "12.1", "11.8"] | ||
env: | ||
GPU_ARCH_TYPE: cuda-manylinux_2_28 | ||
GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | ||
steps: | ||
- name: Purge tools folder (free space for build) | ||
run: rm -rf /opt/hostedtoolcache | ||
- name: Checkout PyTorch builder | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-cuda-aarch64: | ||
runs-on: linux.arm64.2xlarge | ||
strategy: | ||
matrix: | ||
cuda_version: ["12.4"] | ||
env: | ||
GPU_ARCH_TYPE: cuda-aarch64 | ||
GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-rocm: | ||
runs-on: linux.12xlarge.ephemeral | ||
strategy: | ||
matrix: | ||
rocm_version: ["6.0", "6.1"] | ||
env: | ||
GPU_ARCH_TYPE: rocm | ||
GPU_ARCH_VERSION: ${{ matrix.rocm_version }} | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-cpu: | ||
runs-on: ubuntu-22.04 | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-cpu-manylinux_2_28: | ||
runs-on: ubuntu-22.04 | ||
env: | ||
GPU_ARCH_TYPE: cpu-manylinux_2_28 | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-cpu-aarch64: | ||
runs-on: linux.arm64.2xlarge | ||
env: | ||
GPU_ARCH_TYPE: cpu-aarch64 | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-cpu-aarch64-2_28: | ||
runs-on: linux.arm64.2xlarge | ||
env: | ||
GPU_ARCH_TYPE: cpu-aarch64-2_28 | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-cpu-cxx11-abi: | ||
runs-on: ubuntu-22.04 | ||
env: | ||
GPU_ARCH_TYPE: cpu-cxx11-abi | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh | ||
build-docker-xpu: | ||
runs-on: linux.12xlarge.ephemeral | ||
env: | ||
GPU_ARCH_TYPE: xpu | ||
steps: | ||
- name: Checkout PyTorch | ||
uses: actions/checkout@v3 | ||
- name: Authenticate if WITH_PUSH | ||
run: | | ||
if [[ "${WITH_PUSH}" == true ]]; then | ||
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | ||
fi | ||
- name: Build Docker Image | ||
run: | | ||
manywheel/build_docker.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
0.6b | ||
manylinux_2_17 | ||
rocm6.1 | ||
04b5df8c8123f90cba3ede7e971e6fbc6040d506 | ||
77c29fa3f3b614e187d7213d745e989a92708cee2bc6020419ab49019af399d1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
#!/bin/bash | ||
|
||
set -ex | ||
|
||
TARBALL='aotriton.tar.bz2' | ||
# This read command alwasy returns with exit code 1 | ||
read -d "\n" VER MANYLINUX ROCMBASE PINNED_COMMIT SHA256 < aotriton_version.txt || true | ||
ARCH=$(uname -m) | ||
AOTRITON_INSTALL_PREFIX="$1" | ||
AOTRITON_URL="https://github.com/ROCm/aotriton/releases/download/${VER}/aotriton-${VER}-${MANYLINUX}_${ARCH}-${ROCMBASE}-shared.tar.bz2" | ||
|
||
cd "${AOTRITON_INSTALL_PREFIX}" | ||
# Must use -L to follow redirects | ||
curl -L --retry 3 -o "${TARBALL}" "${AOTRITON_URL}" | ||
ACTUAL_SHA256=$(sha256sum "${TARBALL}" | cut -d " " -f 1) | ||
if [ "${SHA256}" != "${ACTUAL_SHA256}" ]; then | ||
echo -n "Error: The SHA256 of downloaded tarball is ${ACTUAL_SHA256}," | ||
echo " which does not match the expected value ${SHA256}." | ||
exit | ||
fi | ||
tar xf "${TARBALL}" && rm -rf "${TARBALL}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
#!/bin/bash | ||
|
||
set -ex | ||
|
||
# Anaconda | ||
# Latest anaconda is using openssl-3 which is incompatible with all currently published versions of git | ||
# Which are using openssl-1.1.1, see https://anaconda.org/anaconda/git/files?version=2.40.1 for example | ||
MINICONDA_URL=https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh | ||
wget -q $MINICONDA_URL | ||
# NB: Manually invoke bash per https://github.com/conda/conda/issues/10431 | ||
bash $(basename "$MINICONDA_URL") -b -p /opt/conda | ||
rm $(basename "$MINICONDA_URL") | ||
export PATH=/opt/conda/bin:$PATH | ||
# See https://github.com/pytorch/builder/issues/1473 | ||
# Pin conda to 23.5.2 as it's the last one compatible with openssl-1.1.1 | ||
conda install -y conda=23.5.2 conda-build anaconda-client git ninja | ||
# The cmake version here needs to match with the minimum version of cmake | ||
# supported by PyTorch (3.18). There is only 3.18.2 on anaconda | ||
/opt/conda/bin/pip3 install cmake==3.18.2 | ||
conda remove -y --force patchelf |
Oops, something went wrong.