Build platform-specific wheels containing libmagic

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

.github/workflows/ci.yml

@@ -19,13 +19,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
+      - name: Copy libmagic into magic dir
+        run: ${{ ( startsWith( matrix.os, 'windows' ) && 'bash add_libmagic.sh' ) || 'sudo -E bash add_libmagic.sh' }}
       - run: pip install --upgrade pip
       - run: pip install --upgrade pytest
       - run: pip install --editable .
-      - if: runner.os == 'macOS'
-        run: brew install libmagic
-      - if: runner.os == 'Windows'
-        run: pip install python-magic-bin
       - run: LC_ALL=en_US.UTF-8 pytest
         shell: bash
         timeout-minutes: 15  # Limit Windows infinite loop.

.github/workflows/wheels.yml

@@ -0,0 +1,141 @@
+name: wheels
+
+on:
+  pull_request:
+  push:
+    branches: master
+  release:
+    types: [released, prereleased]
+  workflow_dispatch:  # allows running workflow manually from the Actions tab
+
+jobs:
+
+  build-sdist:
+    runs-on: ubuntu-latest
+
+    env:
+      PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+
+    - run: sudo apt-get install -y libmagic1
+
+    - name: Build source distribution
+      run: |
+        pip install --upgrade setuptools wheel pip
+        python setup.py sdist
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: dist
+        path: dist/*.tar.*
+
+
+  build-wheels-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      include: ${{ steps.set-matrix.outputs.include }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - run: pip install cibuildwheel==2.17.0  # sync version with pypa/cibuildwheel below
+      - id: set-matrix
+        env:
+          # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch
+          CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*'
+          # skip PyPy wheels for now, and skip i686 wheels because pytest is failing
+          CIBW_SKIP: pp* *i686
+        run: |
+          MATRIX_INCLUDE=$(
+            {
+              cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 |  jq -nRc '{"only": inputs, "os": "macos-13"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch arm64 |  jq -nRc '{"only": inputs, "os": "macos-14"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 |  jq -nRc '{"only": inputs, "os": "windows-latest"}'
+            } | jq -sc
+          )
+          echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
+
+
+  build-wheels:
+    name: build ${{ matrix.only }}
+    needs: build-wheels-matrix
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+
+    - uses: pypa/cibuildwheel@v2.17.0  # sync version with pip install cibuildwheel above
+      timeout-minutes: 10
+      with:
+        only: ${{ matrix.only }}
+      env:
+        CIBW_BUILD_VERBOSITY: 1
+        # add compiled libmagic to the build directory (to include in the wheel)
+        CIBW_BEFORE_BUILD: ${{ ( startsWith( matrix.os, 'macos' ) && 'sudo -E bash add_libmagic.sh' ) || 'bash add_libmagic.sh' }}
+        # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag)
+        MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }}
+        # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic
+        CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'"
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: dist-${{ matrix.only }}
+        path: wheelhouse/*.whl
+
+
+  publish:
+    if: github.event_name == 'release'
+    needs: [build-sdist, build-wheels]
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # softprops/action-gh-release
+      id-token: write  # pypa/gh-action-pypi-publish
+
+    steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+
+    - uses: actions/download-artifact@v4
+      with:
+        path: dist/
+        pattern: dist-*
+        merge-multiple: true
+
+    - run: ls -ltra dist/
+
+    - run: pip install --upgrade python-magic --find-links ./dist
+
+    - name: Smoketest
+      run: python -c "import magic; magic.Magic()"
+
+    - name: Upload release assets
+      uses: softprops/action-gh-release@v0.1.15
+      with:
+        files: dist/*
+
+    - name: Publish package distributions to PyPI
+      uses: pypa/gh-action-pypi-publish@v1.8.14

CHANGELOG

@@ -1,10 +1,14 @@
-Changes to 0.4.29:
+Changes to 0.4.28:
 
+- libmagic and magic.mgc now come bundled in the wheels on PyPI, and will be copied
+  into site-packages/magic along with the Python files of this library
+- magic.loader.load_lib now first searches for libmagic in the same directory as the
+  Python files, then in the current working directory, and only then in standard paths
+- magic.Magic(magic_file=...) and magic.compat.Magic.load(magic_file=...) will now
+  prefer "magic.mgc" in the same directory as the Python files, only if left
+  unspecified by the user (and the MAGIC env var is empty or not set)
 - support MAGIC_SYMLINK (via follow_symlink flag on Magic constructor)
 - correctly throw FileNotFoundException depending on flag
-
-Changes to 0.4.28:
-
 - support "magic-1.dll" on Windows, which is produced by vcpkg
 - add python 3.10 to tox config
 - update test for upstream gzip extensions

README.md

@@ -3,11 +3,11 @@
 [![ci](https://github.com/ahupp/python-magic/actions/workflows/ci.yml/badge.svg)](https://github.com/ahupp/python-magic/actions/workflows/ci.yml)
 [![Join the chat at https://gitter.im/ahupp/python-magic](https://badges.gitter.im/ahupp/python-magic.svg)](https://gitter.im/ahupp/python-magic?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-python-magic is a Python interface to the libmagic file type
-identification library.  libmagic identifies file types by checking
+[python-magic](https://github.com/ahupp/python-magic) is a Python interface to the libmagic file type
+identification library. libmagic identifies file types by checking
 their headers according to a predefined list of file types. This
 functionality is exposed to the command line by the Unix command
-`file`.
+[`file`](https://www.darwinsys.com/file/).
 
 ## Usage
 
@@ -31,8 +31,7 @@ will fail throw if this is attempted.
 ```python
 >>> f = magic.Magic(uncompress=True)
 >>> f.from_file('testdata/test.gz')
-'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28
-21:32:52 2008, from Unix)'
+'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 21:32:52 2008, from Unix)'
 ```
 
 You can also combine the flag options:
@@ -45,27 +44,53 @@ You can also combine the flag options:
 
 ## Installation
 
-The current stable version of python-magic is available on PyPI and
-can be installed by running `pip install python-magic`.
+This module is a simple [CDLL](https://docs.python.org/3/library/ctypes.html) wrapper around the libmagic C library.
+The current stable version of python-magic is available on [PyPI](http://pypi.python.org/pypi/python-magic/)
+and can be installed by running `pip install python-magic`.
 
-Other sources:
+Compiled libmagic and the magic database come bundled in the wheels on PyPI.
+You can use your own `magic.mgc` database by setting the `MAGIC`
+environment variable, or by using `magic.Magic(magic_file='path/to/magic.mgc')`.
+If you want to compile your own libmagic, circumvent the wheels
+by installing from source: `pip install python-magic --no-binary python-magic`.
 
-- PyPI: http://pypi.python.org/pypi/python-magic/
-- GitHub: https://github.com/ahupp/python-magic
+For systems not supported by the wheels, pip installs from source,
+requiring libmagic to be available before installing python-magic:
 
-This module is a simple wrapper around the libmagic C library, and
-that must be installed as well:
+### Linux
 
-### Debian/Ubuntu
+The Linux wheels should run on most systems out of the box.
 
+Depending on your system and CPU architecture, there might be no compatible wheel uploaded.
+However, precompiled libmagic might still be available for your system:
+
+```sh
+# Debian/Ubuntu
+apt-get update && apt-get install -y libmagic1
+# Alpine
+apk add --update libmagic
+# RHEL
+dnf install file-libs
 ```
-sudo apt-get install libmagic1
-```
+
+### Windows
+
+The DLLs that are bundled in the Windows wheels are compiled by @julian-r
+and are hosted at https://github.com/julian-r/file-windows/releases.
+
+For ARM64 Windows, you'll need to compile libmagic from source.
 
 ### OSX
 
-- When using Homebrew: `brew install libmagic`
-- When using macports: `port install file`
+The Mac wheels are compiled with maximum backward compatibility.
+For older Macs, you'll need to install libmagic from source:
+
+```sh
+# homebrew
+brew install libmagic
+# macports
+port install file
+```
 
 If python-magic fails to load the library it may be in a non-standard location, in which case you can set the environment variable `DYLD_LIBRARY_PATH` to point to it.
 
@@ -78,7 +103,7 @@ If python-magic fails to load the library it may be in a non-standard location,
 - 'MagicException: could not find any magic files!': some
   installations of libmagic do not correctly point to their magic
   database file.  Try specifying the path to the file explicitly in the
-  constructor: `magic.Magic(magic_file="path_to_magic_file")`.
+  constructor: `magic.Magic(magic_file='path/to/magic.mgc')`.
 
 - 'WindowsError: [Error 193] %1 is not a valid Win32 application':
   Attempting to run the 32-bit libmagic DLL in a 64-bit build of
@@ -88,7 +113,6 @@ If python-magic fails to load the library it may be in a non-standard location,
 - 'WindowsError: exception: access violation writing 0x00000000 ' This may indicate you are mixing
   Windows Python and Cygwin Python. Make sure your libmagic and python builds are consistent.
 
-
 ## Bug Reports
 
 python-magic is a thin layer over the libmagic C library.

add_libmagic.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+install_source() {
+    # install from source
+    # https://www.darwinsys.com/file/
+    # https://github.com/file/file/blob/FILE5_45/INSTALL#L51
+    (
+        version="file-5.45" &&
+        tmpfile="$(mktemp)" &&
+        curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" &&
+        tar xvf "${tmpfile}" &&
+        cd "${version}" &&
+        ./configure &&
+        make &&
+        make install &&
+        make installcheck &&
+        cd .. &&
+        rm -r "${version}"
+    ) || ( cd .. && false )
+}
+
+install_precompiled() {
+    # Mac https://formulae.brew.sh/formula/libmagic
+    # Debian https://packages.ubuntu.com/libmagic1
+    # Alpine https://pkgs.alpinelinux.org/package/libmagic
+    # RHEL https://git.almalinux.org/rpms/file
+    # Windows https://github.com/julian-r/file-windows
+    if [ -n "$(which brew)" ]; then
+        brew install libmagic
+    elif [ -n "$(which apt-get)" ]; then
+        apt-get update
+        apt-get install -y libmagic1
+    elif [ -n "$(which apk)" ]; then
+        apk add --update libmagic
+    elif [ -n "$(which dnf)" ]; then
+        dnf --setopt install_weak_deps=false -y install file-libs
+    else
+        # windows (no install, just download into current working directory)
+        # could also consider install using `pacman`: https://packages.msys2.org/base/mingw-w64-file
+        # which would require an update of copy_libmagic below to account for new magic.mgc paths
+        python <<EOF
+import platform, sysconfig, io, zipfile, urllib.request
+assert platform.system() == "Windows"
+machine = "x86" if sysconfig.get_platform() == "win32" else "x64"
+url = f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip"
+print("Downloading", url)
+zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(url).read())).extractall(".")
+EOF
+        # check what was copied
+        ls -ltra
+    fi
+}
+
+copy_libmagic() {
+    # on cibuildwheel, the lib needs to exist in the project before running setup.py
+    # copy lib into the magic dir, regardless of platform
+    # this python command relies on current working directory containing `./magic/loader.py`
+    libmagic_path="$(python -c 'from magic.loader import load_lib; print(load_lib()._name)')" &&
+    cp "${libmagic_path}" "magic" &&
+    # only on linux/macos: additionally copy compiled db into magic dir (prefer the one installed by install_source)
+    ( ( cp "/usr/local/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) &&
+    # check what was copied
+    ls -ltra magic
+}
+
+# prefer a recent build from source
+install_source || install_precompiled
+# files to be copied into the wheel
+copy_libmagic

magic/__init__.py

@@ -96,6 +96,12 @@ def __init__(self, mime=False, magic_file=None, mime_encoding=False,
         self.cookie = magic_open(self.flags)
         self.lock = threading.Lock()
 
+        if magic_file is None and not os.environ.get("MAGIC"):
+            # wheels package the mime database in this directory
+            # prefer it when no magic file is specified by the user
+            mime_db = os.path.join(os.path.dirname(__file__), 'magic.mgc')
+            if os.path.exists(mime_db):
+                magic_file = mime_db
         magic_load(self.cookie, magic_file)
 
         # MAGIC_EXTENSION was added in 523 or 524, so bail if