Skip to content

Commit

Permalink
Add sanity checks
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicolasHug
NicolasHug committed Aug 7, 2024
1 parent 794104a commit ddd32b1
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 120 deletions.
225 changes: 113 additions & 112 deletions .github/workflows/wheel.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,125 +35,126 @@ jobs:
echo $CONDA_PREFIX
python --version
python -m pip install --upgrade pip
# - name: Install Pytorch
# run: |
# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
# - name: Check GCC version
# run: |
# # We want to build wheels on gcc 9 to be consistent with the torch
# # binaries, which is why we're using ubuntu-20.04 and not
# # ubuntu-latest. More details below about the GLIBCXX checks.
# if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+"
# then
# echo "Wrong GCC version, exiting!"
# gcc --version
# exit 1
# fi
# - name: Build the torchcodec wheel
# run: |
# # Just for sanity, make sure FFmpeg isn't installed or needed for buidling.
# .github/scripts/assert_ffmpeg_not_installed.sh
- name: Install Pytorch
run: |
python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
- name: Check GCC version
run: |
# We want to build wheels on gcc 9 to be consistent with the torch
# binaries, which is why we're using ubuntu-20.04 and not
# ubuntu-latest. More details below about the GLIBCXX checks.
if ! gcc --version | grep -q -E "9\.[0-9]+\.[0-9]+"
then
echo "Wrong GCC version, exiting!"
gcc --version
exit 1
fi
- name: Build the torchcodec wheel
run: |
# Just for sanity, make sure FFmpeg isn't installed or needed for buidling.
.github/scripts/assert_ffmpeg_not_installed.sh
# python -m pip install build
# BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation
# - name: Validate wheel content
# run: |
# source .github/scripts/helpers.sh
# wheel_path=$(pwd)/$(find dist -type f -name "*.whl")
# echo "Wheel content:"
# unzip -l $wheel_path
python -m pip install build
BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 python -m build . -vvv --no-isolation
- name: Validate wheel content
run: |
source .github/scripts/helpers.sh
wheel_path=$(pwd)/$(find dist -type f -name "*.whl")
echo "Wheel content:"
unzip -l $wheel_path
# for ffmpeg_major_version in 4 5 6 7; do
# assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so
# done
# assert_not_in_wheel $wheel_path libtorchcodec.so
for ffmpeg_major_version in 4 5 6 7; do
assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so
done
assert_not_in_wheel $wheel_path libtorchcodec.so
# for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do
# assert_not_in_wheel $wheel_path $ffmpeg_so
# done
for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do
assert_not_in_wheel $wheel_path $ffmpeg_so
done
# assert_not_in_wheel $wheel_path "^test"
# assert_not_in_wheel $wheel_path "^doc"
# assert_not_in_wheel $wheel_path "^benchmarks"
assert_not_in_wheel $wheel_path "^test"
assert_not_in_wheel $wheel_path "^doc"
assert_not_in_wheel $wheel_path "^benchmarks"
assert_not_in_wheel $wheel_path "^packaging"
# # See invoked python script below for details about this check.
# extracted_wheel_dir=$(mktemp -d)
# unzip -q $wheel_path -d $extracted_wheel_dir
# symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
# python .github/scripts/check_glibcxx.py "$symbols_matches"
# - uses: actions/upload-artifact@v4
# with:
# name: sdist-and-wheel-linux_x86_${{ matrix.python-version }}
# path: dist/*
# See invoked python script below for details about this check.
extracted_wheel_dir=$(mktemp -d)
unzip -q $wheel_path -d $extracted_wheel_dir
symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
python .github/scripts/check_glibcxx.py "$symbols_matches"
- uses: actions/upload-artifact@v4
with:
name: sdist-and-wheel-linux_x86_${{ matrix.python-version }}
path: dist/*

# install-and-test:
# runs-on: ubuntu-latest
# strategy:
# fail-fast: false
# matrix:
# python-version: ['3.8', '3.12']
# ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
# if: ${{ always() }}
# needs: build
# steps:
# - uses: actions/download-artifact@v4
# with:
# name: sdist-and-wheel-linux_x86_${{ matrix.python-version }}
# path: dist/
# - name: Setup conda env
# uses: conda-incubator/setup-miniconda@v2
# with:
# auto-update-conda: true
# miniconda-version: "latest"
# activate-environment: test
# python-version: ${{ matrix.python-version }}
# - name: Update pip
# run: python -m pip install --upgrade pip
# - name: Install PyTorch
# run: |
# python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
# - name: Install torchcodec from the wheel
# run: |
# wheel_path=`find dist -type f -name "*.whl"`
# echo Installing $wheel_path
# python -m pip install $wheel_path -vvv
install-and-test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ['3.8', '3.12']
ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
if: ${{ always() }}
needs: build
steps:
- uses: actions/download-artifact@v4
with:
name: sdist-and-wheel-linux_x86_${{ matrix.python-version }}
path: dist/
- name: Setup conda env
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
miniconda-version: "latest"
activate-environment: test
python-version: ${{ matrix.python-version }}
- name: Update pip
run: python -m pip install --upgrade pip
- name: Install PyTorch
run: |
python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
- name: Install torchcodec from the wheel
run: |
wheel_path=`find dist -type f -name "*.whl"`
echo Installing $wheel_path
python -m pip install $wheel_path -vvv
# - name: Check out repo
# uses: actions/checkout@v3
# - name: Install ffmpeg, post build
# run: |
# # Ideally we would have checked for that before installing the wheel,
# # but we need to checkout the repo to access this file, and we don't
# # want to checkout the repo before installing the wheel to avoid any
# # side-effect. It's OK.
# .github/scripts/assert_ffmpeg_not_installed.sh
- name: Check out repo
uses: actions/checkout@v3
- name: Install ffmpeg, post build
run: |
# Ideally we would have checked for that before installing the wheel,
# but we need to checkout the repo to access this file, and we don't
# want to checkout the repo before installing the wheel to avoid any
# side-effect. It's OK.
.github/scripts/assert_ffmpeg_not_installed.sh
# conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
# ffmpeg -version
conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
ffmpeg -version
# - name: Install test dependencies
# run: |
# python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
# # Ideally we would find a way to get those dependencies from pyproject.toml
# python -m pip install numpy pytest pillow
- name: Install test dependencies
run: |
python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
# Ideally we would find a way to get those dependencies from pyproject.toml
python -m pip install numpy pytest pillow
# - name: Delete the src/ folder just for fun
# run: |
# # The only reason we checked-out the repo is to get access to the
# # tests. We don't care about the rest. Out of precaution, we delete
# # the src/ folder to be extra sure that we're running the code from
# # the installed wheel rather than from the source.
# # This is just to be extra cautious and very overkill because a)
# # there's no way the `torchcodec` package from src/ can be found from
# # the PythonPath: the main point of `src/` is precisely to protect
# # against that and b) if we ever were to execute code from
# # `src/torchcodec`, it would fail loudly because the built .so files
# # aren't present there.
# rm -r src/
# ls
# - name: Smoke test
# run: |
# python test/decoders/manual_smoke_test.py
# - name: Run Python tests
# run: |
# pytest test
- name: Delete the src/ folder just for fun
run: |
# The only reason we checked-out the repo is to get access to the
# tests. We don't care about the rest. Out of precaution, we delete
# the src/ folder to be extra sure that we're running the code from
# the installed wheel rather than from the source.
# This is just to be extra cautious and very overkill because a)
# there's no way the `torchcodec` package from src/ can be found from
# the PythonPath: the main point of `src/` is precisely to protect
# against that and b) if we ever were to execute code from
# `src/torchcodec`, it would fail loudly because the built .so files
# aren't present there.
rm -r src/
ls
- name: Smoke test
run: |
python test/decoders/manual_smoke_test.py
- name: Run Python tests
run: |
pytest test
32 changes: 24 additions & 8 deletions packaging/post_build_script.sh
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,29 @@
#!/bin/bash

echo "LS"
ls
echo "LS dist"
ls dist
source .github/scripts/helpers.sh

wheel_path=$(pwd)/$(find dist -type f -name "*.whl")
echo "Wheel content:"
unzip -l $wheel_path

echo "MVVVV"
mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/')
for ffmpeg_major_version in 4 5 6 7; do
assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so
done
assert_not_in_wheel $wheel_path libtorchcodec.so

for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do
assert_not_in_wheel $wheel_path $ffmpeg_so
done

assert_not_in_wheel $wheel_path "^test"
assert_not_in_wheel $wheel_path "^doc"
assert_not_in_wheel $wheel_path "^benchmarks"
assert_not_in_wheel $wheel_path "^packaging"

echo "LS dist"
ls dist
# See invoked python script below for details about this check.
extracted_wheel_dir=$(mktemp -d)
unzip -q $wheel_path -d $extracted_wheel_dir
symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
python .github/scripts/check_glibcxx.py "$symbols_matches"

mv dist/*linux_x86_64*.whl $(echo dist/*linux_x86_64*.whl | sed 's/linux_x86_64/manylinux_2_17_x86_64.manylinux2014_x86_64/')

0 comments on commit ddd32b1

Please sign in to comment.