diff --git a/.github/workflows/wheel.yaml b/.github/workflows/wheel.yaml index b0534f51..49beaa23 100644 --- a/.github/workflows/wheel.yaml +++ b/.github/workflows/wheel.yaml @@ -35,125 +35,126 @@ jobs: echo $CONDA_PREFIX python --version python -m pip install --upgrade pip - # - name: Install Pytorch - # run: | - # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - # - name: Check GCC version - # run: | - # # We want to build wheels on gcc 9 to be consistent with the torch - # # binaries, which is why we're using ubuntu-20.04 and not - # # ubuntu-latest. More details below about the GLIBCXX checks. - # if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" - # then - # echo "Wrong GCC version, exiting!" - # gcc --version - # exit 1 - # fi - # - name: Build the torchcodec wheel - # run: | - # # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. - # .github/scripts/assert_ffmpeg_not_installed.sh + - name: Install Pytorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Check GCC version + run: | + # We want to build wheels on gcc 9 to be consistent with the torch + # binaries, which is why we're using ubuntu-20.04 and not + # ubuntu-latest. More details below about the GLIBCXX checks. + if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+" + then + echo "Wrong GCC version, exiting!" + gcc --version + exit 1 + fi + - name: Build the torchcodec wheel + run: | + # Just for sanity, make sure FFmpeg isn't installed or needed for buidling. + .github/scripts/assert_ffmpeg_not_installed.sh - # python -m pip install build - # BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation - # - name: Validate wheel content - # run: | - # source .github/scripts/helpers.sh - # wheel_path=$(pwd)/$(find dist -type f -name "*.whl") - # echo "Wheel content:" - # unzip -l $wheel_path + python -m pip install build + BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation + - name: Validate wheel content + run: | + source .github/scripts/helpers.sh + wheel_path=$(pwd)/$(find dist -type f -name "*.whl") + echo "Wheel content:" + unzip -l $wheel_path - # for ffmpeg_major_version in 4 5 6 7; do - # assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so - # done - # assert_not_in_wheel $wheel_path libtorchcodec.so + for ffmpeg_major_version in 4 5 6 7; do + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so + done + assert_not_in_wheel $wheel_path libtorchcodec.so - # for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do - # assert_not_in_wheel $wheel_path $ffmpeg_so - # done + for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do + assert_not_in_wheel $wheel_path $ffmpeg_so + done - # assert_not_in_wheel $wheel_path "^test" - # assert_not_in_wheel $wheel_path "^doc" - # assert_not_in_wheel $wheel_path "^benchmarks" + assert_not_in_wheel $wheel_path "^test" + assert_not_in_wheel $wheel_path "^doc" + assert_not_in_wheel $wheel_path "^benchmarks" + assert_not_in_wheel $wheel_path "^packaging" - # # See invoked python script below for details about this check. - # extracted_wheel_dir=$(mktemp -d) - # unzip -q $wheel_path -d $extracted_wheel_dir - # symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) - # python .github/scripts/check_glibcxx.py "$symbols_matches" - # - uses: actions/upload-artifact@v4 - # with: - # name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - # path: dist/* + # See invoked python script below for details about this check. + extracted_wheel_dir=$(mktemp -d) + unzip -q $wheel_path -d $extracted_wheel_dir + symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) + python .github/scripts/check_glibcxx.py "$symbols_matches" + - uses: actions/upload-artifact@v4 + with: + name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + path: dist/* - # install-and-test: - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # matrix: - # python-version: ['3.8', '3.12'] - # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] - # if: ${{ always() }} - # needs: build - # steps: - # - uses: actions/download-artifact@v4 - # with: - # name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} - # path: dist/ - # - name: Setup conda env - # uses: conda-incubator/setup-miniconda@v2 - # with: - # auto-update-conda: true - # miniconda-version: "latest" - # activate-environment: test - # python-version: ${{ matrix.python-version }} - # - name: Update pip - # run: python -m pip install --upgrade pip - # - name: Install PyTorch - # run: | - # python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - # - name: Install torchcodec from the wheel - # run: | - # wheel_path=`find dist -type f -name "*.whl"` - # echo Installing $wheel_path - # python -m pip install $wheel_path -vvv + install-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.12'] + ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] + if: ${{ always() }} + needs: build + steps: + - uses: actions/download-artifact@v4 + with: + name: sdist-and-wheel-linux_x86_${{ matrix.python-version }} + path: dist/ + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install --upgrade pip + - name: Install PyTorch + run: | + python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install torchcodec from the wheel + run: | + wheel_path=`find dist -type f -name "*.whl"` + echo Installing $wheel_path + python -m pip install $wheel_path -vvv - # - name: Check out repo - # uses: actions/checkout@v3 - # - name: Install ffmpeg, post build - # run: | - # # Ideally we would have checked for that before installing the wheel, - # # but we need to checkout the repo to access this file, and we don't - # # want to checkout the repo before installing the wheel to avoid any - # # side-effect. It's OK. - # .github/scripts/assert_ffmpeg_not_installed.sh + - name: Check out repo + uses: actions/checkout@v3 + - name: Install ffmpeg, post build + run: | + # Ideally we would have checked for that before installing the wheel, + # but we need to checkout the repo to access this file, and we don't + # want to checkout the repo before installing the wheel to avoid any + # side-effect. It's OK. + .github/scripts/assert_ffmpeg_not_installed.sh - # conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge - # ffmpeg -version + conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge + ffmpeg -version - # - name: Install test dependencies - # run: | - # python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu - # # Ideally we would find a way to get those dependencies from pyproject.toml - # python -m pip install numpy pytest pillow + - name: Install test dependencies + run: | + python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu + # Ideally we would find a way to get those dependencies from pyproject.toml + python -m pip install numpy pytest pillow - # - name: Delete the src/ folder just for fun - # run: | - # # The only reason we checked-out the repo is to get access to the - # # tests. We don't care about the rest. Out of precaution, we delete - # # the src/ folder to be extra sure that we're running the code from - # # the installed wheel rather than from the source. - # # This is just to be extra cautious and very overkill because a) - # # there's no way the `torchcodec` package from src/ can be found from - # # the PythonPath: the main point of `src/` is precisely to protect - # # against that and b) if we ever were to execute code from - # # `src/torchcodec`, it would fail loudly because the built .so files - # # aren't present there. - # rm -r src/ - # ls - # - name: Smoke test - # run: | - # python test/decoders/manual_smoke_test.py - # - name: Run Python tests - # run: | - # pytest test + - name: Delete the src/ folder just for fun + run: | + # The only reason we checked-out the repo is to get access to the + # tests. We don't care about the rest. Out of precaution, we delete + # the src/ folder to be extra sure that we're running the code from + # the installed wheel rather than from the source. + # This is just to be extra cautious and very overkill because a) + # there's no way the `torchcodec` package from src/ can be found from + # the PythonPath: the main point of `src/` is precisely to protect + # against that and b) if we ever were to execute code from + # `src/torchcodec`, it would fail loudly because the built .so files + # aren't present there. + rm -r src/ + ls + - name: Smoke test + run: | + python test/decoders/manual_smoke_test.py + - name: Run Python tests + run: | + pytest test diff --git a/packaging/post_build_script.sh b/packaging/post_build_script.sh index a8aa6c46..f256b50c 100755 --- a/packaging/post_build_script.sh +++ b/packaging/post_build_script.sh @@ -1,13 +1,29 @@ #!/bin/bash -echo "LS" -ls -echo "LS dist" -ls dist +source .github/scripts/helpers.sh +wheel_path=$(pwd)/$(find dist -type f -name "*.whl") +echo "Wheel content:" +unzip -l $wheel_path -echo "MVVVV" -mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/') +for ffmpeg_major_version in 4 5 6 7; do + assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so +done +assert_not_in_wheel $wheel_path libtorchcodec.so + +for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do + assert_not_in_wheel $wheel_path $ffmpeg_so +done + +assert_not_in_wheel $wheel_path "^test" +assert_not_in_wheel $wheel_path "^doc" +assert_not_in_wheel $wheel_path "^benchmarks" +assert_not_in_wheel $wheel_path "^packaging" -echo "LS dist" -ls dist +# See invoked python script below for details about this check. +extracted_wheel_dir=$(mktemp -d) +unzip -q $wheel_path -d $extracted_wheel_dir +symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.) +python .github/scripts/check_glibcxx.py "$symbols_matches" + +mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/')