From c4091e6cb800f92a60848b6b2747e7077034d740 Mon Sep 17 00:00:00 2001 From: Yushion Yang <2685310462@qq.com> Date: Fri, 23 Oct 2026 19:01:22 +0800 Subject: [PATCH] Ignore a leading UTF-8 BOM in gitignore lines --- CHANGES_1.in.rst | 1 + pathspec/gitignore.py | 17 ++++++- tests/test_06_gitignore.py | 90 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/CHANGES_1.in.rst b/CHANGES_1.in.rst index e2a76ae..af69eb0 100644 --- a/CHANGES_1.in.rst +++ b/CHANGES_1.in.rst @@ -21,6 +21,7 @@ New features: Bug fixes: +- `GitIgnoreSpec.from_lines()` now ignores a UTF-8 byte order mark at the beginning of the first line, matching Git's handling of BOM-prefixed ignore files. - Hyperscan backend no longer fails on multibyte UTF-8 characters. - `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`. - `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions. diff --git a/pathspec/gitignore.py b/pathspec/gitignore.py index 9e8e6f8..dadf982 100644 --- a/pathspec/gitignore.py +++ b/pathspec/gitignore.py @@ -10,6 +10,8 @@ from collections.abc import ( Iterable, Sequence) +from itertools import ( + chain) from typing import ( Callable, # Replaced by `collections.abc.Callable` in 3.9.2. Optional, # Replaced by `X | None` in 3.10. @@ -107,7 +109,8 @@ def from_lines( # type: ignore *lines* (:class:`~collections.abc.Iterable`) yields each uncompiled pattern (:class:`str`). This simply has to yield each line, so it can be a :class:`io.TextIOBase` (e.g., from :func:`open` or :class:`io.StringIO`) or - the result from :meth:`str.splitlines`. + the result from :meth:`str.splitlines`. A UTF-8 byte order mark at the + beginning of the first line is ignored, as in Git. *pattern_factory* does not need to be set for :class:`GitIgnoreSpec`. If set, it should be either ``"gitignore"`` or :class:`.GitIgnoreSpecPattern`. @@ -147,6 +150,18 @@ def from_lines( # type: ignore f"results." )) # TypeError + if _is_iterable(lines): + # A BOM belongs to the ignore file, not to each individual pattern. + # Preserve it on later lines and strip at most one from the first line. + line_iter = iter(lines) + first_line = next(line_iter, None) + if first_line is not None: + if isinstance(first_line, str): + first_line = first_line.removeprefix('\ufeff') + elif isinstance(first_line, bytes): + first_line = first_line.removeprefix(b'\xef\xbb\xbf') + lines = chain((first_line,), line_iter) + self = super().from_lines(use_factory, lines, backend=backend, _test_backend_factory=_test_backend_factory) # type: ignore[arg-type,type-var] return self # type: ignore[return-value] diff --git a/tests/test_06_gitignore.py b/tests/test_06_gitignore.py index 523e905..abb0da6 100644 --- a/tests/test_06_gitignore.py +++ b/tests/test_06_gitignore.py @@ -2,6 +2,7 @@ This script tests :class:`.GitIgnoreSpec`. """ +import io import itertools from collections.abc import ( Iterable, @@ -34,6 +35,8 @@ GitIgnoreSpec) from pathspec.pattern import ( Pattern) +from pathspec.patterns.gitignore.spec import ( + GitIgnoreSpecPattern) from pathspec._typing import ( AnyStr) # Removed in 3.18. @@ -179,6 +182,93 @@ def _sub_test( yield _sub_test +class GitIgnoreSpecBomTest(GitIgnoreSpecMixin, TestCase): + """ + Test Git's handling of a UTF-8 byte order mark at the start of an ignore file. + """ + + def test_pattern(self): + for sub_test in self.parameterize_from_lines(['\ufeff*.log\n']): + with sub_test() as spec: + self.assertTrue(spec.match_file('test.log')) + self.assertTrue(spec.match_file('dir/test.log')) + self.assertFalse(spec.match_file('test.txt')) + + def test_comment(self): + for sub_test in self.parameterize_from_lines(['\ufeff# comment\n', '*.log\n']): + with sub_test() as spec: + self.assertFalse(spec.match_file('\ufeff# comment')) + self.assertTrue(spec.match_file('test.log')) + + def test_negation(self): + for sub_test in self.parameterize_from_lines(['\ufeff!keep.log\n']): + with sub_test() as spec: + self.assertIs(spec.check_file('keep.log').include, False) + + def test_root_directory(self): + for sub_test in self.parameterize_from_lines(['\ufeff/build/\n']): + with sub_test() as spec: + self.assertTrue(spec.match_file('build/output.txt')) + self.assertFalse(spec.match_file('dir/build/output.txt')) + + def test_bom_only_line(self): + for sub_test in self.parameterize_from_lines(['\ufeff\n', '*.log\n']): + with sub_test() as spec: + self.assertFalse(spec.match_file('\ufeff')) + self.assertTrue(spec.match_file('test.log')) + + def test_later_bom_is_literal(self): + for lines in (['\n', '\ufeff*.log\n'], ['', '\ufeff*.log\n']): + for sub_test in self.parameterize_from_lines(lines): + with sub_test() as spec: + self.assertFalse(spec.match_file('test.log')) + self.assertTrue(spec.match_file('\ufefftest.log')) + + def test_strip_only_one_bom(self): + for sub_test in self.parameterize_from_lines(['\ufeff\ufeff*.log\n']): + with sub_test() as spec: + self.assertFalse(spec.match_file('test.log')) + self.assertTrue(spec.match_file('\ufefftest.log')) + + def test_stream_and_iterator(self): + for ending in ('\n', '\r\n'): + content = f'\ufeff*.log{ending}!keep.log{ending}' + for lines in (io.StringIO(content), iter(content.splitlines(keepends=True))): + with self.subTest(ending=ending, lines=lines): + spec = GitIgnoreSpec.from_lines(lines, backend='simple') + self.assertTrue(spec.match_file('test.log')) + self.assertFalse(spec.match_file('keep.log')) + + def test_bytes(self): + for lines, expected in ( + ([b'\xef\xbb\xbf*.log\n'], [b'*.log\n']), + ([b'\n', b'\xef\xbb\xbf*.log\n'], [b'\n', b'\xef\xbb\xbf*.log\n']), + ([b'\xef\xbb\xbf\xef\xbb\xbf*.log\n'], [b'\xef\xbb\xbf*.log\n']), + ): + with self.subTest(lines=lines): + spec = GitIgnoreSpec.from_lines(lines, backend='simple') + patterns = [GitIgnoreSpecPattern(line) for line in expected] + self.assertEqual(spec.patterns, patterns) + + def test_reversed_args(self): + for factory in ('gitignore', GitIgnoreSpecPattern): + with self.subTest(factory=factory): + spec = GitIgnoreSpec.from_lines(factory, ['\ufeff*.log'], backend='simple') + self.assertTrue(spec.match_file('test.log')) + + def test_empty_input(self): + for lines in ([], iter([]), io.StringIO('')): + with self.subTest(lines=lines): + spec = GitIgnoreSpec.from_lines(lines, backend='simple') + self.assertEqual(spec.patterns, []) + + def test_invalid_lines(self): + for lines in (None, '*.log', b'*.log'): + with self.subTest(lines=lines): + with self.assertRaises(TypeError): + GitIgnoreSpec.from_lines(lines, backend='simple') + + class GitIgnoreSpecTest(GitIgnoreSpecMixin, TestCase): """ The :class:`GitIgnoreSpecTest` class tests the :class:`.GitIgnoreSpec` class.