Ignore a leading UTF-8 BOM in gitignore lines

CHANGES_1.in.rst

@@ -21,6 +21,7 @@ New features:
 
 Bug fixes:
 
+- `GitIgnoreSpec.from_lines()` now ignores a UTF-8 byte order mark at the beginning of the first line, matching Git's handling of BOM-prefixed ignore files.
 - Hyperscan backend no longer fails on multibyte UTF-8 characters.
 - `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
 - `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions.

pathspec/gitignore.py

@@ -10,6 +10,8 @@
 from collections.abc import (
 	Iterable,
 	Sequence)
+from itertools import (
+	chain)
 from typing import (
 	Callable,  # Replaced by `collections.abc.Callable` in 3.9.2.
 	Optional,  # Replaced by `X | None` in 3.10.
@@ -107,7 +109,8 @@ def from_lines(  # type: ignore
 		*lines* (:class:`~collections.abc.Iterable`) yields each uncompiled pattern
 		(:class:`str`). This simply has to yield each line, so it can be a
 		:class:`io.TextIOBase` (e.g., from :func:`open` or :class:`io.StringIO`) or
-		the result from :meth:`str.splitlines`.
+		the result from :meth:`str.splitlines`. A UTF-8 byte order mark at the
+		beginning of the first line is ignored, as in Git.
 
 		*pattern_factory* does not need to be set for :class:`GitIgnoreSpec`. If
 		set, it should be either ``"gitignore"`` or :class:`.GitIgnoreSpecPattern`.
@@ -147,6 +150,18 @@ def from_lines(  # type: ignore
 				f"results."
 			))  # TypeError
 
+		if _is_iterable(lines):
+			# A BOM belongs to the ignore file, not to each individual pattern.
+			# Preserve it on later lines and strip at most one from the first line.
+			line_iter = iter(lines)
+			first_line = next(line_iter, None)
+			if first_line is not None:
+				if isinstance(first_line, str):
+					first_line = first_line.removeprefix('\ufeff')
+				elif isinstance(first_line, bytes):
+					first_line = first_line.removeprefix(b'\xef\xbb\xbf')
+				lines = chain((first_line,), line_iter)
+
 		self = super().from_lines(use_factory, lines, backend=backend, _test_backend_factory=_test_backend_factory)  # type: ignore[arg-type,type-var]
 		return self  # type: ignore[return-value]
 

tests/test_06_gitignore.py

@@ -2,6 +2,7 @@
 This script tests :class:`.GitIgnoreSpec`.
 """
 
+import io
 import itertools
 from collections.abc import (
 	Iterable,
@@ -34,6 +35,8 @@
 	GitIgnoreSpec)
 from pathspec.pattern import (
 	Pattern)
+from pathspec.patterns.gitignore.spec import (
+	GitIgnoreSpecPattern)
 from pathspec._typing import (
 	AnyStr)  # Removed in 3.18.
 
@@ -179,6 +182,93 @@ def _sub_test(
 			yield _sub_test
 
 
+class GitIgnoreSpecBomTest(GitIgnoreSpecMixin, TestCase):
+	"""
+	Test Git's handling of a UTF-8 byte order mark at the start of an ignore file.
+	"""
+
+	def test_pattern(self):
+		for sub_test in self.parameterize_from_lines(['\ufeff*.log\n']):
+			with sub_test() as spec:
+				self.assertTrue(spec.match_file('test.log'))
+				self.assertTrue(spec.match_file('dir/test.log'))
+				self.assertFalse(spec.match_file('test.txt'))
+
+	def test_comment(self):
+		for sub_test in self.parameterize_from_lines(['\ufeff# comment\n', '*.log\n']):
+			with sub_test() as spec:
+				self.assertFalse(spec.match_file('\ufeff# comment'))
+				self.assertTrue(spec.match_file('test.log'))
+
+	def test_negation(self):
+		for sub_test in self.parameterize_from_lines(['\ufeff!keep.log\n']):
+			with sub_test() as spec:
+				self.assertIs(spec.check_file('keep.log').include, False)
+
+	def test_root_directory(self):
+		for sub_test in self.parameterize_from_lines(['\ufeff/build/\n']):
+			with sub_test() as spec:
+				self.assertTrue(spec.match_file('build/output.txt'))
+				self.assertFalse(spec.match_file('dir/build/output.txt'))
+
+	def test_bom_only_line(self):
+		for sub_test in self.parameterize_from_lines(['\ufeff\n', '*.log\n']):
+			with sub_test() as spec:
+				self.assertFalse(spec.match_file('\ufeff'))
+				self.assertTrue(spec.match_file('test.log'))
+
+	def test_later_bom_is_literal(self):
+		for lines in (['\n', '\ufeff*.log\n'], ['', '\ufeff*.log\n']):
+			for sub_test in self.parameterize_from_lines(lines):
+				with sub_test() as spec:
+					self.assertFalse(spec.match_file('test.log'))
+					self.assertTrue(spec.match_file('\ufefftest.log'))
+
+	def test_strip_only_one_bom(self):
+		for sub_test in self.parameterize_from_lines(['\ufeff\ufeff*.log\n']):
+			with sub_test() as spec:
+				self.assertFalse(spec.match_file('test.log'))
+				self.assertTrue(spec.match_file('\ufefftest.log'))
+
+	def test_stream_and_iterator(self):
+		for ending in ('\n', '\r\n'):
+			content = f'\ufeff*.log{ending}!keep.log{ending}'
+			for lines in (io.StringIO(content), iter(content.splitlines(keepends=True))):
+				with self.subTest(ending=ending, lines=lines):
+					spec = GitIgnoreSpec.from_lines(lines, backend='simple')
+					self.assertTrue(spec.match_file('test.log'))
+					self.assertFalse(spec.match_file('keep.log'))
+
+	def test_bytes(self):
+		for lines, expected in (
+			([b'\xef\xbb\xbf*.log\n'], [b'*.log\n']),
+			([b'\n', b'\xef\xbb\xbf*.log\n'], [b'\n', b'\xef\xbb\xbf*.log\n']),
+			([b'\xef\xbb\xbf\xef\xbb\xbf*.log\n'], [b'\xef\xbb\xbf*.log\n']),
+		):
+			with self.subTest(lines=lines):
+				spec = GitIgnoreSpec.from_lines(lines, backend='simple')
+				patterns = [GitIgnoreSpecPattern(line) for line in expected]
+				self.assertEqual(spec.patterns, patterns)
+
+	def test_reversed_args(self):
+		for factory in ('gitignore', GitIgnoreSpecPattern):
+			with self.subTest(factory=factory):
+				spec = GitIgnoreSpec.from_lines(factory, ['\ufeff*.log'], backend='simple')
+				self.assertTrue(spec.match_file('test.log'))
+
+	def test_empty_input(self):
+		for lines in ([], iter([]), io.StringIO('')):
+			with self.subTest(lines=lines):
+				spec = GitIgnoreSpec.from_lines(lines, backend='simple')
+				self.assertEqual(spec.patterns, [])
+
+	def test_invalid_lines(self):
+		for lines in (None, '*.log', b'*.log'):
+			with self.subTest(lines=lines):
+				with self.assertRaises(TypeError):
+					GitIgnoreSpec.from_lines(lines, backend='simple')
+
+
 class GitIgnoreSpecTest(GitIgnoreSpecMixin, TestCase):
 	"""
 	The :class:`GitIgnoreSpecTest` class tests the :class:`.GitIgnoreSpec` class.