Preserve descendant restrictions in explicit trailing globstar patterns

CHANGES.rst

@@ -20,6 +20,8 @@ New features:
 
 Bug fixes:
 
+- Preserve the descendant-directory restriction in explicit trailing ``/**/`` patterns, including wildcard prefixes and collapsed recursive wildcards.
+
 - Honor `on_error` when opening a directory fails during tree traversal.
 
 - `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.

README.rst

@@ -94,6 +94,13 @@ You do not specify the style of pattern for ``GitIgnoreSpec`` because it should
 always use ``GitIgnoreSpecPattern`` internally.
 
 
+A pattern ending in ``/**/`` matches descendant directories and their contents,
+while leaving the named parent and its immediate files available. For example,
+``folder/**/`` does not ignore ``folder/`` or ``folder/file``, but does ignore
+``folder/sub/`` and ``folder/sub/file``. An ordinary ``folder/`` pattern continues
+to ignore the parent directory and everything inside it.
+
+
 Performance
 -----------
 

pathspec/patterns/gitignore/basic.py

@@ -39,13 +39,18 @@ class GitIgnoreBasicPattern(_GitIgnoreBasePattern):
 	def __normalize_segments(
 		is_dir_pattern: bool,
 		pattern_segs: list[str],
+		*,
+		is_recursive_dir_pattern: bool = False,
 	) -> tuple[Optional[list[str]], Optional[str]]:
 		"""
 		Normalize the pattern segments to make processing easier.
 
 		*is_dir_pattern* (:class:`bool`) is whether the pattern is a directory
 		pattern (i.e., ends with a slash '/').
 
+		*is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern
+		ends with an explicit double-asterisk segment followed by a slash (``**/``).
+
 		*pattern_segs* (:class:`list` of :class:`str`) contains the pattern
 		segments. This may be modified in place.
 
@@ -121,6 +126,8 @@ def __normalize_segments(
 			and pattern_segs[0] == '**'
 			and pattern_segs[1] == '*'
 			and pattern_segs[2] == '**'
+			and is_dir_pattern
+			and not is_recursive_dir_pattern
 		):
 			# The pattern "*/" will be normalized to "**/*/**" which will match every
 			# file not in the root directory. Special case this pattern for
@@ -226,6 +233,7 @@ def pattern_to_regex(
 		# Check whether the pattern is specifically a directory pattern before
 		# normalization.
 		is_dir_pattern = not orig_segs[-1]
+		is_recursive_dir_pattern = is_dir_pattern and orig_segs[-2] == '**'
 
 		if pattern_str == '/':
 			# EDGE CASE: A single slash ('/') is not addressed by the gitignore
@@ -239,6 +247,7 @@ def pattern_to_regex(
 		try:
 			pattern_segs, override_regex = cls.__normalize_segments(
 				is_dir_pattern, orig_segs,
+				is_recursive_dir_pattern=is_recursive_dir_pattern,
 			)
 		except ValueError as e:
 			raise GitIgnorePatternError((
@@ -254,6 +263,7 @@ def pattern_to_regex(
 			try:
 				regex_parts = cls.__translate_segments(
 					seg_errors, is_dir_pattern, pattern_segs,
+					is_recursive_dir_pattern=is_recursive_dir_pattern,
 				)
 			except (_PosixClassError, _RangeNotationError) as e:
 				# EDGE CASE: Git discards patterns with an invalid range notation or an
@@ -297,6 +307,8 @@ def __translate_segments(
 		errors: Literal['literal', 'raise'],
 		is_dir_pattern: bool,
 		pattern_segs: list[str],
+		*,
+		is_recursive_dir_pattern: bool = False,
 	) -> list[str]:
 		"""
 		Translate the pattern segments to regular expressions.
@@ -311,6 +323,9 @@ def __translate_segments(
 		*is_dir_pattern* (:class:`bool`) is whether the original pattern ends
 		with a slash.
 
+		*is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern
+		ends with an explicit double-asterisk segment followed by a slash (``**/``).
+
 		*pattern_segs* (:class:`list` of :class:`str`) contains the pattern
 		segments.
 
@@ -348,7 +363,11 @@ def __translate_segments(
 					assert i == end, (i, end)
 					# A normalized pattern ending with double-asterisks ('**') will match
 					# nonempty trailing path segments, not the parent directory itself.
-					out_parts.append('/' if is_dir_pattern else '/[^/]')
+					if is_recursive_dir_pattern:
+						# An explicit trailing **/ requires a descendant directory.
+						out_parts.append('/(?s:.+/)?[^/]+/')
+					else:
+						out_parts.append('/' if is_dir_pattern else '/[^/]')
 
 			else:
 				# Match path segment.

pathspec/patterns/gitignore/spec.py

@@ -69,13 +69,18 @@ class GitIgnoreSpecPattern(_GitIgnoreBasePattern):
 	def __normalize_segments(
 		is_dir_pattern: bool,
 		pattern_segs: list[str],
+		*,
+		is_recursive_dir_pattern: bool = False,
 	) -> tuple[Optional[list[str]], Optional[str]]:
 		"""
 		Normalize the pattern segments to make processing easier.
 
 		*is_dir_pattern* (:class:`bool`) is whether the pattern is a directory
 		pattern (i.e., ends with a slash '/').
 
+		*is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern
+		ends with an explicit double-asterisk segment followed by a slash (``**/``).
+
 		*pattern_segs* (:class:`list` of :class:`str`) contains the pattern
 		segments. This may be modified in place.
 
@@ -151,14 +156,13 @@ def __normalize_segments(
 			and pattern_segs[0] == '**'
 			and pattern_segs[1] == '*'
 			and pattern_segs[2] == '**'
+			and is_dir_pattern
+			and not is_recursive_dir_pattern
 		):
 			# The pattern "*/" will be normalized to "**/*/**" which will match every
 			# file not in the root directory. Special case this pattern for
 			# efficiency.
-			if is_dir_pattern:
-				return (None, _DIR_MARK_CG)
-			else:
-				return (None, '/')
+			return (None, _DIR_MARK_CG)
 
 		# No regular expression override, return modified pattern segments.
 		return (pattern_segs, None)
@@ -264,11 +268,13 @@ def pattern_to_regex(
 		# Check whether the pattern is specifically a directory pattern before
 		# normalization.
 		is_dir_pattern = not orig_segs[-1]
+		is_recursive_dir_pattern = is_dir_pattern and orig_segs[-2] == '**'
 
 		# Normalize pattern to make processing easier.
 		try:
 			pattern_segs, override_regex = cls.__normalize_segments(
 				is_dir_pattern, orig_segs,
+				is_recursive_dir_pattern=is_recursive_dir_pattern,
 			)
 		except ValueError as e:
 			raise GitIgnorePatternError((
@@ -284,6 +290,7 @@ def pattern_to_regex(
 			try:
 				regex_parts = cls.__translate_segments(
 					seg_errors, is_dir_pattern, pattern_segs,
+					is_recursive_dir_pattern=is_recursive_dir_pattern,
 				)
 			except (_PosixClassError, _RangeNotationError) as e:
 				if errors == 'raise':
@@ -325,6 +332,8 @@ def __translate_segments(
 		errors: Literal['literal', 'raise'],
 		is_dir_pattern: bool,
 		pattern_segs: list[str],
+		*,
+		is_recursive_dir_pattern: bool = False,
 	) -> list[str]:
 		"""
 		Translate the pattern segments to regular expressions.
@@ -339,6 +348,9 @@ def __translate_segments(
 		*is_dir_pattern* (:class:`bool`) is whether the pattern is a directory
 		pattern (i.e., ends with a slash '/').
 
+		*is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern
+		ends with an explicit double-asterisk segment followed by a slash (``**/``).
+
 		*pattern_segs* (:class:`list` of :class:`str`) contains the pattern
 		segments.
 
@@ -374,7 +386,11 @@ def __translate_segments(
 					assert i == end, (i, end)
 					# A normalized pattern ending with double-asterisks ('**') will match
 					# nonempty trailing path segments, not the parent directory itself.
-					if is_dir_pattern:
+					if is_recursive_dir_pattern:
+						# An explicit trailing **/ matches descendant directories,
+						# not the parent or files immediately inside the parent.
+						out_parts.append(f'/(?s:.+/)?[^/]+{_DIR_MARK_CG}')
+					elif is_dir_pattern:
 						out_parts.append(_DIR_MARK_CG)
 					else:
 						out_parts.append('/[^/]')

tests/test_10_globstar_directories.py

@@ -0,0 +1,110 @@
+"""
+Tests trailing recursive wildcards restricted to descendant directories.
+"""
+
+import re
+from itertools import product
+from unittest import TestCase
+
+from pathspec.patterns.gitignore.basic import GitIgnoreBasicPattern
+from pathspec.patterns.gitignore.spec import GitIgnoreSpecPattern
+
+from .test_06_gitignore import GitIgnoreSpecMixin
+
+
+class GlobstarDirectoryPatternTest(TestCase):
+	"""
+	Tests literal, anchored and wildcard prefixes in both pattern implementations.
+	"""
+
+	def test_descendant_directories(self):
+		"""
+		A trailing **/ requires a descendant directory, not the parent or a direct file.
+		"""
+		cases = [
+			('folder/**/', ['folder/sub/', 'folder/sub/file', 'folder/sub/deep/file'],
+			 ['folder/', 'folder/file', 'other/folder/sub/file']),
+			('/folder/**/**/', ['folder/sub/', 'folder/sub/file'],
+			 ['folder/', 'folder/file', 'other/folder/sub/file']),
+			('**/folder/**/', ['folder/sub/file', 'other/folder/sub/file'],
+			 ['folder/', 'folder/file', 'other/folder/', 'other/folder/file']),
+			('*/**/', ['folder/sub/', 'folder/sub/file', 'other/sub/deep/file'],
+			 ['folder/', 'folder/file', 'other/', 'other/file']),
+			('**/*/**/', ['folder/sub/', 'folder/sub/file', 'other/sub/deep/file'],
+			 ['folder/', 'folder/file', 'other/', 'other/file']),
+			('folder/*/**/', ['folder/sub/deep/', 'folder/sub/deep/file'],
+			 ['folder/', 'folder/file', 'folder/sub/', 'folder/sub/file']),
+		]
+		for pattern_class, (text, matches, misses), binary in product(
+			(GitIgnoreBasicPattern, GitIgnoreSpecPattern), cases, (False, True),
+		):
+			with self.subTest(pattern_class=pattern_class, text=text, binary=binary):
+				pattern_text = text.encode() if binary else text
+				regex, include = pattern_class.pattern_to_regex(pattern_text)
+				self.assertIs(include, True)
+				compiled = re.compile(regex)
+				for file in matches:
+					self.assertIsNotNone(compiled.search(file.encode() if binary else file), file)
+				for file in misses:
+					self.assertIsNone(compiled.search(file.encode() if binary else file), file)
+
+	def test_explicit_globstar_contents_shortcut(self):
+		"""
+		The explicit **/*/** pattern is not the automatic directory-only */ shortcut.
+		"""
+		for pattern_class in (GitIgnoreBasicPattern, GitIgnoreSpecPattern):
+			pattern = pattern_class('**/*/**')
+			self.assertIsNone(pattern.match_file('folder/'))
+			self.assertIsNone(pattern.match_file('file'))
+			self.assertIsNotNone(pattern.match_file('folder/file'))
+			self.assertIsNotNone(pattern.match_file('folder/sub/file'))
+
+	def test_regular_directory_and_recursive_shortcuts(self):
+		"""
+		Ordinary directory patterns and root-wide recursive shortcuts keep their meaning.
+		"""
+		for pattern_class, text in product(
+			(GitIgnoreBasicPattern, GitIgnoreSpecPattern), ('folder/', '**/', '**/**/'),
+		):
+			with self.subTest(pattern_class=pattern_class, text=text):
+				pattern = pattern_class(text)
+				for file in ('folder/', 'folder/file', 'folder/sub/file'):
+					self.assertIsNotNone(pattern.match_file(file), file)
+
+	def test_newline_directory_names(self):
+		"""
+		Recursive directory matching includes newline characters within names.
+		"""
+		for pattern_class in (GitIgnoreBasicPattern, GitIgnoreSpecPattern):
+			pattern = pattern_class('folder/**/')
+			for file in ('folder/line\nbreak/', 'folder/line\nbreak/file'):
+				self.assertIsNotNone(pattern.match_file(file), file)
+			self.assertIsNone(pattern.match_file('folder/line\nbreak'))
+
+
+class GlobstarDirectorySpecTest(GitIgnoreSpecMixin, TestCase):
+	"""
+	Tests all matching backend and expression-order configurations.
+	"""
+
+	def test_literal_prefix(self):
+		"""
+		A parent and its immediate files remain available for traversal.
+		"""
+		for begin in self.parameterize_from_lines(['folder/**/']):
+			with begin() as spec:
+				for file in ('folder/', 'folder/file'):
+					self.assertFalse(spec.match_file(file), file)
+				for file in ('folder/sub/', 'folder/sub/file', 'folder/sub/deep/file'):
+					self.assertTrue(spec.match_file(file), file)
+
+	def test_wildcard_prefix(self):
+		"""
+		Wildcard prefixes still require at least one additional directory.
+		"""
+		for begin in self.parameterize_from_lines(['*/**/']):
+			with begin() as spec:
+				for file in ('folder/', 'folder/file', 'other/file'):
+					self.assertFalse(spec.match_file(file), file)
+				for file in ('folder/sub/file', 'other/sub/deep/file'):
+					self.assertTrue(spec.match_file(file), file)