From ec952d7633856aa495ef46104d8cd3eed9ee74be Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Thu, 7 Sep 2023 12:09:12 +0200 Subject: [PATCH] Build platform-specific wheels containing libmagic --- .github/workflows/main.yml | 134 +++++++++++++++++++++++++++++++++++++ Makefile | 20 ++++++ README.md | 37 ++++++---- magic/__init__.py | 7 +- magic/loader.py | 73 +++++++++++--------- setup.py | 29 +++++++- 6 files changed, 255 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 Makefile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..771bcd4 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,134 @@ +name: GH + +permissions: + contents: write + +on: + pull_request: + push: + branches: master + release: + types: [released, prereleased] + workflow_dispatch: # allows running workflow manually from the Actions tab + +jobs: + + build-sdist: + runs-on: ubuntu-latest + + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - run: sudo apt-get install -y libmagic1 + + - name: Build source distribution + run: | + pip install -U setuptools wheel pip + python setup.py sdist + + - uses: actions/upload-artifact@v3 + with: + name: dist + path: dist/*.tar.* + + + build-wheels-matrix: + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + - run: pip install cibuildwheel==2.15.0 + - id: set-matrix + env: + CIBW_PROJECT_REQUIRES_PYTHON: '==3.8.*' + run: | + MATRIX_INCLUDE=$( + { + cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64,i686 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp | jq -nRc '{"only": inputs, "os": "macos-11"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp | jq -nRc '{"only": inputs, "os": "windows-latest"}' + } | jq -sc + ) + echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT + + + build-wheels: + needs: build-wheels-matrix + runs-on: ${{ matrix.os }} + name: Build ${{ matrix.only }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v2 + + - uses: pypa/cibuildwheel@v2.15.0 + timeout-minutes: 10 + with: + only: ${{ matrix.only }} + env: + CIBW_BUILD_VERBOSITY: 1 + CIBW_BEFORE_BUILD: 'bash -c "make install_libmagic"' + + - uses: actions/upload-artifact@v3 + with: + name: dist + path: wheelhouse/*.whl + + + publish: + needs: [build-sdist, build-wheels] + if: github.event_name == 'release' + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-python@v4 + with: + python-version: 3.x + + - uses: actions/download-artifact@v3 + with: + name: dist + path: dist/ + + - run: ls -ltra dist/ + + - run: pip install -U twine python-magic --find-links ./dist + + - name: Smoketest + run: python -c "import magic; magic.Magic()" + + - name: Upload release assets + uses: softprops/action-gh-release@v0.1.15 + with: + files: dist/* + + - name: Upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + twine upload dist/* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cd6b92e --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +SHELL := /bin/bash + +.PHONY: install_libmagic +## Install libmagic +install_libmagic: + # Debian https://packages.ubuntu.com/libmagic1 + # RHEL https://git.almalinux.org/rpms/file + # Mac https://formulae.brew.sh/formula/libmagic + # Windows https://github.com/julian-r/file-windows + ( ( ( brew install libmagic || ( apt-get update && apt-get install -y libmagic1 ) ) || apk add --update libmagic ) || yum install file-libs ) || ( python -c 'import platform, sysconfig, io, zipfile, urllib.request; assert platform.system() == "Windows"; machine = "x86" if sysconfig.get_platform() == "win32" else "x64"; print(machine); zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip").read())).extractall(".")' && ls -ltra ) + # on cibuildwheel, the lib needs to exist in the project before running setup.py + python -c "import subprocess; from magic.loader import load_lib; lib = load_lib()._name; print(f'linking {lib}'); subprocess.check_call(['cp', lib, 'magic'])" + cp /usr/share/misc/magic.mgc magic || true # only on linux + ls -ltra magic + +.DEFAULT_GOAL := help +.PHONY: help +## Print Makefile documentation +help: + @perl -0 -nle 'printf("\033[36m %-15s\033[0m %s\n", "$$2", "$$1") while m/^##\s*([^\r\n]+)\n^([\w.-]+):[^=]/gm' $(MAKEFILE_LIST) | sort diff --git a/README.md b/README.md index fb1bc0e..2d22ec8 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,7 @@ will fail throw if this is attempted. ```python >>> f = magic.Magic(uncompress=True) >>> f.from_file('testdata/test.gz') -'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 -21:32:52 2008, from Unix)' +'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 21:32:52 2008, from Unix)' ``` You can also combine the flag options: @@ -53,26 +52,40 @@ Other sources: - GitHub: https://github.com/ahupp/python-magic This module is a simple wrapper around the libmagic C library, and -that must be installed as well: +comes bundled in the wheels on PyPI. For systems not supported by the wheels, libmagic +needs to be installed before installing this library: -### Debian/Ubuntu +### Linux -``` -sudo apt-get install libmagic1 +The Linux wheels should run on most systems out of the box. + +Depending on your system and CPU architecture, there might be no compatible wheel uploaded. However, precompiled libmagic might still be available for your system: + +```sh +# Debian/Ubuntu +apt-get update && apt-get install -y libmagic1 +# Alpine +apk add --update libmagic +# RHEL +yum install file-libs ``` ### Windows -You'll need DLLs for libmagic. @julian-r maintains a pypi package with the DLLs, you can fetch it with: +The DLLs that are bundled in the Windows wheels are compiled by @julian-r and hosted at https://github.com/julian-r/file-windows/releases. -``` -pip install python-magic-bin -``` +For ARM64 Windows, you'll need to compile libmagic from source. ### OSX -- When using Homebrew: `brew install libmagic` -- When using macports: `port install file` +The Mac wheels are compiled on GitHub Actions using `macos-11` runners. For older Macs, you'll need to install libmagic from source: + +```sh +# homebrew +brew install libmagic +# macports +port install file +``` ### Troubleshooting diff --git a/magic/__init__.py b/magic/__init__.py index d05ebf9..552fe9e 100644 --- a/magic/__init__.py +++ b/magic/__init__.py @@ -330,7 +330,12 @@ def magic_descriptor(cookie, fd): def magic_load(cookie, filename): - return _magic_load(cookie, coerce_filename(filename)) + try: + return _magic_load(cookie, coerce_filename(filename)) + except MagicException: + # wheels package the mime database in this directory + filename = os.path.join(os.path.dirname(__file__), 'magic.mgc') + return _magic_load(cookie, coerce_filename(filename)) magic_setflags = libmagic.magic_setflags diff --git a/magic/loader.py b/magic/loader.py index 228a35c..8ece31d 100644 --- a/magic/loader.py +++ b/magic/loader.py @@ -1,50 +1,61 @@ from ctypes.util import find_library import ctypes -import sys import glob import os.path +import subprocess +import sys def _lib_candidates(): + here = os.path.dirname(__file__) - yield find_library('magic') + if sys.platform == 'darwin': - if sys.platform == 'darwin': + paths = [ + here, + os.path.abspath("."), + '/opt/local/lib', + '/usr/local/lib', + '/opt/homebrew/lib', + ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') - paths = [ - '/opt/local/lib', - '/usr/local/lib', - '/opt/homebrew/lib', - ] + glob.glob('/usr/local/Cellar/libmagic/*/lib') + for i in paths: + yield os.path.join(i, 'libmagic.dylib') - for i in paths: - yield os.path.join(i, 'libmagic.dylib') + elif sys.platform in ('win32', 'cygwin'): - elif sys.platform in ('win32', 'cygwin'): + prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] - prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1'] + for i in prefixes: + # find_library searches in %PATH% but not the current directory, + # so look for both + yield os.path.join(here, '%s.dll' % i) + yield os.path.join(os.path.abspath("."), '%s.dll' % i) + yield find_library(i) - for i in prefixes: - # find_library searches in %PATH% but not the current directory, - # so look for both - yield './%s.dll' % (i,) - yield find_library(i) + elif sys.platform == 'linux': + # on some linux systems (musl/alpine), find_library('magic') returns None + yield subprocess.check_output( + "( ldconfig -p | grep 'libmagic.so.1' | grep -o '/.*' ) || echo '/usr/lib/libmagic.so.1'", + shell=True, + universal_newlines=True, + ).strip() + yield os.path.join(here, 'libmagic.so.1') + yield os.path.join(os.path.abspath("."), 'libmagic.so.1') - elif sys.platform == 'linux': - # This is necessary because alpine is bad - yield 'libmagic.so.1' + yield find_library('magic') def load_lib(): - for lib in _lib_candidates(): - # find_library returns None when lib not found - if lib is None: - continue - try: - return ctypes.CDLL(lib) - except OSError: - pass - else: - # It is better to raise an ImportError since we are importing magic module - raise ImportError('failed to find libmagic. Check your installation') + for lib in _lib_candidates(): + # find_library returns None when lib not found + if lib is None: + continue + try: + return ctypes.CDLL(lib) + except OSError as exc: + pass + else: + # It is better to raise an ImportError since we are importing magic module + raise ImportError('failed to find libmagic. Check your installation') diff --git a/setup.py b/setup.py index d98b731..cab8dd0 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,11 @@ import setuptools import io import os +import sys +# python packages should not install succesfully if libraries are missing +from magic.loader import load_lib +lib = load_lib()._name def read(file_name): """Read a text file and return the content as a string.""" @@ -12,6 +16,28 @@ def read(file_name): encoding='utf-8') as f: return f.read() +def get_cmdclass(): + """Build a forward compatible ABI3 wheel when `setup.py bdist_wheel` is called.""" + if sys.version_info[0] == 2: + return {} + + try: + from wheel.bdist_wheel import bdist_wheel + except ImportError: + return {} + + class bdist_wheel_abi3(bdist_wheel): + def get_tag(self): + python, abi, _ = super().get_tag() + # get the platform tag based on libmagic included in this wheel + self.root_is_pure = False + _, _, plat = super().get_tag() + return python, abi, plat + + return {"bdist_wheel": bdist_wheel_abi3} + +cmdclass = get_cmdclass() + setuptools.setup( name='python-magic', description='File type identification using libmagic', @@ -23,8 +49,9 @@ def read(file_name): long_description_content_type='text/markdown', packages=['magic'], package_data={ - 'magic': ['py.typed', '*.pyi', '**/*.pyi'], + 'magic': ['py.typed', '*.pyi', '*.dylib*', '*.dll', '*.so*', 'magic.mgc'] }, + cmdclass=cmdclass, keywords="mime magic file", license="MIT", python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',