From ab3ee0de09fa51e9a1c9c64d7634b076a1f36d12 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:32:23 +0200 Subject: [PATCH 1/5] Build libmagic from source on windows Run add_libmagic.sh in MSYS2 like on linux and macos, instead of bundling the MSYS2 file package: current MSYS2 no longer ships mingw-w64-i686-file, its last build crashes (0xC0000409) on any magic_file call, and 5.47 breaks from_buffer ref https://github.com/ahupp/python-magic/issues/362 Link the UCRT like python does (ucrt64, and -mcrtdll=ucrt on win32), which fixes from_descriptor (Bad file descriptor). Bundle only the DLLs libmagic actually imports (ntldd -R), instead of the flattened package dependency tree. --- .github/workflows/wheels.yml | 44 ++++++++++-------------------------- README.md | 2 +- add_libmagic.sh | 11 +++++---- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 867d2e90..c37e6357 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -89,40 +89,16 @@ jobs: if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 - # For Windows, grabbing latest file from MSYS2 is easier than building from source - # It's generally up to date ref https://packages.msys2.org/base/mingw-w64-file - - name: Setup MSYS2 and install file + # For Windows, add_libmagic.sh builds libmagic with the MSYS2 toolchain (CIBW_BEFORE_BUILD_WINDOWS below) + # ucrt64 over mingw64: libmagic shares the C runtime with python, so from_descriptor works + - name: Setup MSYS2 if: runner.os == 'Windows' - uses: msys2/setup-msys2@v2.27.0 + uses: msys2/setup-msys2@v2.33.0 with: - msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }} - location: D:\ - install: >- - ${{ endsWith(matrix.only, '32') && 'mingw-w64-i686-file' || 'mingw-w64-x86_64-file' }} - - # The DLL dependency tree flattened out ref "Dependencies" https://packages.msys2.org/packages/mingw-w64-x86_64-file - - name: Copy Windows ddl and mgc - if: runner.os == 'Windows' - run: | - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/share/misc/magic.mgc" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libmagic-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libsystre-0.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libtre-5.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libasprintf-0.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libintl-8.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libatomic-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libgomp-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libquadmath-0.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libstdc++-6.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libcharset-1.dll" "magic" - cp "/msys64/${{ endsWith(matrix.only, '32') && 'mingw32' || 'mingw64' }}/bin/libiconv-2.dll" "magic" - - # These are needed additionally in the win32 wheel ref https://packages.msys2.org/packages/mingw-w64-i686-file - - name: Copy additional 32-bit runtime DLLs - if: runner.os == 'Windows' && endsWith(matrix.only, '32') - run: | - cp "/msys64/mingw32/bin/libgcc_s_dw2-1.dll" "magic" - cp "/msys64/mingw32/bin/libwinpthread-1.dll" "magic" + msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'ucrt64' }} + path-type: inherit # add_libmagic.sh needs the cibuildwheel python + install: make + pacboy: cc:p libsystre:p ntldd:p - uses: pypa/cibuildwheel@v2.17.0 # sync version with pip install cibuildwheel above timeout-minutes: 10 @@ -133,6 +109,10 @@ jobs: # add compiled libmagic to the build directory (to include in the wheel) CIBW_BEFORE_BUILD_MACOS: sudo -E bash add_libmagic.sh CIBW_BEFORE_BUILD_LINUX: bash add_libmagic.sh + CIBW_BEFORE_BUILD_WINDOWS: msys2 -c "bash add_libmagic.sh" + # windows: prefer the MSYS2 toolchain over C:\mingw64 on PATH, CFLAGS like https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-file/PKGBUILD + # (win32: link the UCRT like python does, as there is no 32-bit ucrt MSYS2 environment) + CIBW_ENVIRONMENT_WINDOWS: CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag) MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }} # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic diff --git a/README.md b/README.md index deb06fe0..90e008dc 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ dnf install file-libs ### Windows -The DLLs that are bundled in the Windows wheels are built by MSYS2 ([mingw32](https://packages.msys2.org/packages/mingw-w64-i686-file) and [mingw64](https://packages.msys2.org/packages/mingw-w64-x86_64-file)). +The DLLs that are bundled in the Windows wheels are built from source with the MSYS2 toolchain (mingw32 and ucrt64). For ARM64 Windows, you'll need to compile libmagic from source or [install](https://packages.msys2.org/packages/mingw-w64-clang-aarch64-file) from MSYS2. diff --git a/add_libmagic.sh b/add_libmagic.sh index b1133ff3..7ccd2b65 100755 --- a/add_libmagic.sh +++ b/add_libmagic.sh @@ -2,6 +2,9 @@ set -euxo pipefail +# on windows (MSYS2), install next to the DLLs libmagic depends on +prefix="${MINGW_PREFIX:-/usr/local}" + install_source() { # install from source # https://www.darwinsys.com/file/ @@ -12,7 +15,7 @@ install_source() { curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" && tar xvf "${tmpfile}" && cd "${version}" && - ./configure && + ./configure --prefix="${prefix}" && make && make install && make installcheck && @@ -44,14 +47,14 @@ copy_libmagic() { # this python command relies on current working directory containing `./magic/loader.py` libmagic_path="$(python -c 'from magic.loader import load_lib; print(load_lib()._name)')" && cp "${libmagic_path}" "magic" && + # on windows (MSYS2), also copy the DLLs libmagic depends on + ( [ -z "${MINGW_PREFIX:-}" ] || ntldd -R "${libmagic_path}" | awk '{print $3}' | cygpath -u -f - | grep "^${MINGW_PREFIX}/bin/" | xargs cp -t "magic" ) && # additionally copy compiled db into magic dir (prefer the one installed by install_source) - ( ( cp "/usr/local/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) && + ( ( cp "${prefix}/share/misc/magic.mgc" "magic" || cp "/usr/share/misc/magic.mgc" "magic" ) || true ) && # check what was copied ls -ltra magic } -# skip windows (taken care of separately in wheels.yml) -python -c 'import platform; assert platform.system() != "Windows"' || ( echo "skipping on windows" && exit 0 ) # prefer a recent build from source install_source || install_precompiled # files to be copied into the wheel From f0f18b0bdd9a851a9bf0c76f847cc79e7c3a8f9e Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:32:34 +0200 Subject: [PATCH 2/5] Run test suite against all wheels Replaces the inline smoke test, which moves into the suite as test_from_buffer_hls_mp4 (also guards https://github.com/ahupp/python-magic/issues/362). On Windows: check out test data without CRLF conversion, and skip what libmagic does not support there (symlinks, non-ANSI paths). --- .gitattributes | 2 ++ .github/workflows/wheels.yml | 8 +++++--- test/python_magic_test.py | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..62cb0ace --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# keep test files byte-identical on Windows checkouts (no CRLF conversion) +test/testdata/** -text diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c37e6357..2e48e22d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -112,11 +112,13 @@ jobs: CIBW_BEFORE_BUILD_WINDOWS: msys2 -c "bash add_libmagic.sh" # windows: prefer the MSYS2 toolchain over C:\mingw64 on PATH, CFLAGS like https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-file/PKGBUILD # (win32: link the UCRT like python does, as there is no 32-bit ucrt MSYS2 environment) - CIBW_ENVIRONMENT_WINDOWS: CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" + CIBW_ENVIRONMENT_WINDOWS: LC_ALL=en_US.UTF-8 CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag) MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }} - # simple smoke test run on each wheel: this is an HLS MP4 video, only recognised in recent versions of libmagic - CIBW_TEST_COMMAND: python -c "import magic; assert magic.Magic(mime=True).from_buffer(b'\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00') == 'video/mp4'" + # run the test suite against the installed wheel (CIBW_ENVIRONMENT_WINDOWS overrides, so it repeats LC_ALL) + CIBW_ENVIRONMENT: LC_ALL=en_US.UTF-8 + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest --import-mode=importlib {project}/test - uses: actions/upload-artifact@v4 with: diff --git a/test/python_magic_test.py b/test/python_magic_test.py index 651c0be7..ca730f8c 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -210,6 +210,9 @@ def test_all_cases(self): os.environ["TZ"] = "UTC" try: for filename, cases in CASES.items(): + if sys.platform == "win32" and filename == b"\xce\xbb": + # libmagic opens paths with the narrow (ANSI codepage) api on Windows + continue filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename) print("test case ", filename, file=sys.stderr) for flag_variants, outputs in cases: @@ -282,6 +285,11 @@ def test_getparam(self): except NotImplementedError: pass + def test_from_buffer_hls_mp4(self): + # HLS MP4 video, only recognised in recent versions of libmagic (and broken in 5.47 ref #362) + buf = b"\x00\x00\x00\x1cftypiso5\x00\x00\x00\x01isomiso5hlsf\x00\x00" + self.assertEqual("video/mp4", magic.from_buffer(buf, mime=True)) + def test_name_count(self): m = magic.Magic() with open(os.path.join(self.TESTDATA_DIR, "name_use.jpg"), "rb") as f: @@ -296,6 +304,7 @@ def test_pathlike(self): m = magic.Magic(mime=True) self.assertEqual("application/pdf", m.from_file(path)) + @unittest.skipIf(sys.platform == "win32", "libmagic has no symlink support on Windows") def test_symlink(self): # TODO: 3.0 if not hasattr(tempfile, "TemporaryDirectory"): From 322673307f8141655fc7e9cd9f5e84aa7a777600 Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:32:46 +0200 Subject: [PATCH 3/5] Build linux i686 wheels libmagic 5.46 built from source passes the test suite on manylinux2014_i686 and musllinux_1_1_i686. --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2e48e22d..b5e5116b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -56,8 +56,8 @@ jobs: env: # only mention one (trivial) python version, as py2.py3 wheels only need to be build once per arch CIBW_PROJECT_REQUIRES_PYTHON: '==3.12.*' - # skip PyPy wheels for now, and skip i686 wheels because pytest is failing - CIBW_SKIP: pp* *i686 + # skip PyPy wheels for now + CIBW_SKIP: pp* run: | MATRIX_INCLUDE=$( { From 13a59427b698c7477547a346a45e3cb87fdc3edb Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:32:46 +0200 Subject: [PATCH 4/5] Build windows arm64 wheels Built natively on the windows-11-arm runner with the MSYS2 clangarm64 toolchain. --- .github/workflows/wheels.yml | 8 +++++--- README.md | 4 +--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b5e5116b..e8166917 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -64,7 +64,8 @@ jobs: cibuildwheel --print-build-identifiers --platform linux --arch all | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ && cibuildwheel --print-build-identifiers --platform macos --arch x86_64 | jq -nRc '{"only": inputs, "os": "macos-15-intel"}' \ && cibuildwheel --print-build-identifiers --platform macos --arch arm64 | jq -nRc '{"only": inputs, "os": "macos-15"}' \ - && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | jq -nRc '{"only": inputs, "os": "windows-latest"}' + && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | jq -nRc '{"only": inputs, "os": "windows-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch ARM64 | jq -nRc '{"only": inputs, "os": "windows-11-arm"}' } | jq -sc ) echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT @@ -95,7 +96,7 @@ jobs: if: runner.os == 'Windows' uses: msys2/setup-msys2@v2.33.0 with: - msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || 'ucrt64' }} + msystem: ${{ endsWith(matrix.only, '32') && 'mingw32' || endsWith(matrix.only, 'arm64') && 'clangarm64' || 'ucrt64' }} path-type: inherit # add_libmagic.sh needs the cibuildwheel python install: make pacboy: cc:p libsystre:p ntldd:p @@ -112,7 +113,8 @@ jobs: CIBW_BEFORE_BUILD_WINDOWS: msys2 -c "bash add_libmagic.sh" # windows: prefer the MSYS2 toolchain over C:\mingw64 on PATH, CFLAGS like https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-file/PKGBUILD # (win32: link the UCRT like python does, as there is no 32-bit ucrt MSYS2 environment) - CIBW_ENVIRONMENT_WINDOWS: LC_ALL=en_US.UTF-8 CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" + # and skip the x86-only libtool file format check so the tarball's libtool links arm64 DLLs + CIBW_ENVIRONMENT_WINDOWS: LC_ALL=en_US.UTF-8 CC=cc CFLAGS="-O2 -Wno-incompatible-pointer-types${{ endsWith(matrix.only, '32') && ' -mcrtdll=ucrt -D_UCRT' || '' }}" lt_cv_deplibs_check_method=pass_all # build macos wheels with maximum backwards compatibility (gcc -mmacosx-version-min flag) MACOSX_DEPLOYMENT_TARGET: ${{ ( endsWith( matrix.only, 'arm64' ) && '11.0' ) || '10.9' }} # run the test suite against the installed wheel (CIBW_ENVIRONMENT_WINDOWS overrides, so it repeats LC_ALL) diff --git a/README.md b/README.md index 90e008dc..35121878 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,7 @@ dnf install file-libs ### Windows -The DLLs that are bundled in the Windows wheels are built from source with the MSYS2 toolchain (mingw32 and ucrt64). - -For ARM64 Windows, you'll need to compile libmagic from source or [install](https://packages.msys2.org/packages/mingw-w64-clang-aarch64-file) from MSYS2. +The DLLs that are bundled in the Windows wheels are built from source with the MSYS2 toolchain (mingw32, ucrt64 and clangarm64). ### OSX From 5fe336bbea166a5ef0c27b898f0252267b2354dc Mon Sep 17 00:00:00 2001 From: ddelange <14880945+ddelange@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:27:45 +0200 Subject: [PATCH 5/5] Bump libmagic to 5.48 5.48 fixes the from_buffer regression of 5.47 ref https://github.com/ahupp/python-magic/issues/362 (guarded by test_from_buffer_hls_mp4). 5.48 identifies parquet files (application/vnd.apache.parquet, Apache Parquet file). --- add_libmagic.sh | 4 ++-- test/python_magic_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/add_libmagic.sh b/add_libmagic.sh index 7ccd2b65..cea1a842 100755 --- a/add_libmagic.sh +++ b/add_libmagic.sh @@ -8,9 +8,9 @@ prefix="${MINGW_PREFIX:-/usr/local}" install_source() { # install from source # https://www.darwinsys.com/file/ - # https://github.com/file/file/blob/FILE5_46/INSTALL#L51 + # https://github.com/file/file/blob/FILE5_48/INSTALL#L51 ( - version="file-5.46" && + version="file-5.48" && tmpfile="$(mktemp)" && curl -sSLo "${tmpfile}" "https://astron.com/pub/file/${version}.tar.gz" && tar xvf "${tmpfile}" && diff --git a/test/python_magic_test.py b/test/python_magic_test.py index ca730f8c..9b6db8af 100755 --- a/test/python_magic_test.py +++ b/test/python_magic_test.py @@ -93,8 +93,8 @@ class TestFile: (NO_SOFT, ["data"]), ], b"test.snappy.parquet": [ - (COMMON_MIME, ["application/octet-stream"]), - (COMMON_PLAIN, ["data", "Apache Parquet", "Par archive data"]), + (COMMON_MIME, ["application/octet-stream", "application/vnd.apache.parquet"]), + (COMMON_PLAIN, ["data", "Apache Parquet", "Apache Parquet file", "Par archive data"]), (NO_SOFT, ["data"]), ], b"test.json": [