From 1d503ccf060c0cb3b633d67848e534974d6faf77 Mon Sep 17 00:00:00 2001 From: Cheruku Sri Charan Reddy <185475071+sricharanreddycheruku@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:15:24 +0000 Subject: [PATCH] Preserve descendant restrictions in explicit trailing globstar patterns --- CHANGES.rst | 2 + README.rst | 7 ++ pathspec/patterns/gitignore/basic.py | 21 ++++- pathspec/patterns/gitignore/spec.py | 26 ++++-- tests/test_10_globstar_directories.py | 110 ++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 tests/test_10_globstar_directories.py diff --git a/CHANGES.rst b/CHANGES.rst index b3e7efc..4429411 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,8 @@ New features: Bug fixes: +- Preserve the descendant-directory restriction in explicit trailing ``/**/`` patterns, including wildcard prefixes and collapsed recursive wildcards. + - Honor `on_error` when opening a directory fails during tree traversal. - `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`. diff --git a/README.rst b/README.rst index 9824fb9..4dcffb1 100644 --- a/README.rst +++ b/README.rst @@ -94,6 +94,13 @@ You do not specify the style of pattern for ``GitIgnoreSpec`` because it should always use ``GitIgnoreSpecPattern`` internally. +A pattern ending in ``/**/`` matches descendant directories and their contents, +while leaving the named parent and its immediate files available. For example, +``folder/**/`` does not ignore ``folder/`` or ``folder/file``, but does ignore +``folder/sub/`` and ``folder/sub/file``. An ordinary ``folder/`` pattern continues +to ignore the parent directory and everything inside it. + + Performance ----------- diff --git a/pathspec/patterns/gitignore/basic.py b/pathspec/patterns/gitignore/basic.py index f396a71..3e66e0a 100644 --- a/pathspec/patterns/gitignore/basic.py +++ b/pathspec/patterns/gitignore/basic.py @@ -39,6 +39,8 @@ class GitIgnoreBasicPattern(_GitIgnoreBasePattern): def __normalize_segments( is_dir_pattern: bool, pattern_segs: list[str], + *, + is_recursive_dir_pattern: bool = False, ) -> tuple[Optional[list[str]], Optional[str]]: """ Normalize the pattern segments to make processing easier. @@ -46,6 +48,9 @@ def __normalize_segments( *is_dir_pattern* (:class:`bool`) is whether the pattern is a directory pattern (i.e., ends with a slash '/'). + *is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern + ends with an explicit double-asterisk segment followed by a slash (``**/``). + *pattern_segs* (:class:`list` of :class:`str`) contains the pattern segments. This may be modified in place. @@ -121,6 +126,8 @@ def __normalize_segments( and pattern_segs[0] == '**' and pattern_segs[1] == '*' and pattern_segs[2] == '**' + and is_dir_pattern + and not is_recursive_dir_pattern ): # The pattern "*/" will be normalized to "**/*/**" which will match every # file not in the root directory. Special case this pattern for @@ -226,6 +233,7 @@ def pattern_to_regex( # Check whether the pattern is specifically a directory pattern before # normalization. is_dir_pattern = not orig_segs[-1] + is_recursive_dir_pattern = is_dir_pattern and orig_segs[-2] == '**' if pattern_str == '/': # EDGE CASE: A single slash ('/') is not addressed by the gitignore @@ -239,6 +247,7 @@ def pattern_to_regex( try: pattern_segs, override_regex = cls.__normalize_segments( is_dir_pattern, orig_segs, + is_recursive_dir_pattern=is_recursive_dir_pattern, ) except ValueError as e: raise GitIgnorePatternError(( @@ -254,6 +263,7 @@ def pattern_to_regex( try: regex_parts = cls.__translate_segments( seg_errors, is_dir_pattern, pattern_segs, + is_recursive_dir_pattern=is_recursive_dir_pattern, ) except (_PosixClassError, _RangeNotationError) as e: # EDGE CASE: Git discards patterns with an invalid range notation or an @@ -297,6 +307,8 @@ def __translate_segments( errors: Literal['literal', 'raise'], is_dir_pattern: bool, pattern_segs: list[str], + *, + is_recursive_dir_pattern: bool = False, ) -> list[str]: """ Translate the pattern segments to regular expressions. @@ -311,6 +323,9 @@ def __translate_segments( *is_dir_pattern* (:class:`bool`) is whether the original pattern ends with a slash. + *is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern + ends with an explicit double-asterisk segment followed by a slash (``**/``). + *pattern_segs* (:class:`list` of :class:`str`) contains the pattern segments. @@ -348,7 +363,11 @@ def __translate_segments( assert i == end, (i, end) # A normalized pattern ending with double-asterisks ('**') will match # nonempty trailing path segments, not the parent directory itself. - out_parts.append('/' if is_dir_pattern else '/[^/]') + if is_recursive_dir_pattern: + # An explicit trailing **/ requires a descendant directory. + out_parts.append('/(?s:.+/)?[^/]+/') + else: + out_parts.append('/' if is_dir_pattern else '/[^/]') else: # Match path segment. diff --git a/pathspec/patterns/gitignore/spec.py b/pathspec/patterns/gitignore/spec.py index a443868..5ef8341 100644 --- a/pathspec/patterns/gitignore/spec.py +++ b/pathspec/patterns/gitignore/spec.py @@ -69,6 +69,8 @@ class GitIgnoreSpecPattern(_GitIgnoreBasePattern): def __normalize_segments( is_dir_pattern: bool, pattern_segs: list[str], + *, + is_recursive_dir_pattern: bool = False, ) -> tuple[Optional[list[str]], Optional[str]]: """ Normalize the pattern segments to make processing easier. @@ -76,6 +78,9 @@ def __normalize_segments( *is_dir_pattern* (:class:`bool`) is whether the pattern is a directory pattern (i.e., ends with a slash '/'). + *is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern + ends with an explicit double-asterisk segment followed by a slash (``**/``). + *pattern_segs* (:class:`list` of :class:`str`) contains the pattern segments. This may be modified in place. @@ -151,14 +156,13 @@ def __normalize_segments( and pattern_segs[0] == '**' and pattern_segs[1] == '*' and pattern_segs[2] == '**' + and is_dir_pattern + and not is_recursive_dir_pattern ): # The pattern "*/" will be normalized to "**/*/**" which will match every # file not in the root directory. Special case this pattern for # efficiency. - if is_dir_pattern: - return (None, _DIR_MARK_CG) - else: - return (None, '/') + return (None, _DIR_MARK_CG) # No regular expression override, return modified pattern segments. return (pattern_segs, None) @@ -264,11 +268,13 @@ def pattern_to_regex( # Check whether the pattern is specifically a directory pattern before # normalization. is_dir_pattern = not orig_segs[-1] + is_recursive_dir_pattern = is_dir_pattern and orig_segs[-2] == '**' # Normalize pattern to make processing easier. try: pattern_segs, override_regex = cls.__normalize_segments( is_dir_pattern, orig_segs, + is_recursive_dir_pattern=is_recursive_dir_pattern, ) except ValueError as e: raise GitIgnorePatternError(( @@ -284,6 +290,7 @@ def pattern_to_regex( try: regex_parts = cls.__translate_segments( seg_errors, is_dir_pattern, pattern_segs, + is_recursive_dir_pattern=is_recursive_dir_pattern, ) except (_PosixClassError, _RangeNotationError) as e: if errors == 'raise': @@ -325,6 +332,8 @@ def __translate_segments( errors: Literal['literal', 'raise'], is_dir_pattern: bool, pattern_segs: list[str], + *, + is_recursive_dir_pattern: bool = False, ) -> list[str]: """ Translate the pattern segments to regular expressions. @@ -339,6 +348,9 @@ def __translate_segments( *is_dir_pattern* (:class:`bool`) is whether the pattern is a directory pattern (i.e., ends with a slash '/'). + *is_recursive_dir_pattern* (:class:`bool`) is whether the original pattern + ends with an explicit double-asterisk segment followed by a slash (``**/``). + *pattern_segs* (:class:`list` of :class:`str`) contains the pattern segments. @@ -374,7 +386,11 @@ def __translate_segments( assert i == end, (i, end) # A normalized pattern ending with double-asterisks ('**') will match # nonempty trailing path segments, not the parent directory itself. - if is_dir_pattern: + if is_recursive_dir_pattern: + # An explicit trailing **/ matches descendant directories, + # not the parent or files immediately inside the parent. + out_parts.append(f'/(?s:.+/)?[^/]+{_DIR_MARK_CG}') + elif is_dir_pattern: out_parts.append(_DIR_MARK_CG) else: out_parts.append('/[^/]') diff --git a/tests/test_10_globstar_directories.py b/tests/test_10_globstar_directories.py new file mode 100644 index 0000000..ad28d69 --- /dev/null +++ b/tests/test_10_globstar_directories.py @@ -0,0 +1,110 @@ +""" +Tests trailing recursive wildcards restricted to descendant directories. +""" + +import re +from itertools import product +from unittest import TestCase + +from pathspec.patterns.gitignore.basic import GitIgnoreBasicPattern +from pathspec.patterns.gitignore.spec import GitIgnoreSpecPattern + +from .test_06_gitignore import GitIgnoreSpecMixin + + +class GlobstarDirectoryPatternTest(TestCase): + """ + Tests literal, anchored and wildcard prefixes in both pattern implementations. + """ + + def test_descendant_directories(self): + """ + A trailing **/ requires a descendant directory, not the parent or a direct file. + """ + cases = [ + ('folder/**/', ['folder/sub/', 'folder/sub/file', 'folder/sub/deep/file'], + ['folder/', 'folder/file', 'other/folder/sub/file']), + ('/folder/**/**/', ['folder/sub/', 'folder/sub/file'], + ['folder/', 'folder/file', 'other/folder/sub/file']), + ('**/folder/**/', ['folder/sub/file', 'other/folder/sub/file'], + ['folder/', 'folder/file', 'other/folder/', 'other/folder/file']), + ('*/**/', ['folder/sub/', 'folder/sub/file', 'other/sub/deep/file'], + ['folder/', 'folder/file', 'other/', 'other/file']), + ('**/*/**/', ['folder/sub/', 'folder/sub/file', 'other/sub/deep/file'], + ['folder/', 'folder/file', 'other/', 'other/file']), + ('folder/*/**/', ['folder/sub/deep/', 'folder/sub/deep/file'], + ['folder/', 'folder/file', 'folder/sub/', 'folder/sub/file']), + ] + for pattern_class, (text, matches, misses), binary in product( + (GitIgnoreBasicPattern, GitIgnoreSpecPattern), cases, (False, True), + ): + with self.subTest(pattern_class=pattern_class, text=text, binary=binary): + pattern_text = text.encode() if binary else text + regex, include = pattern_class.pattern_to_regex(pattern_text) + self.assertIs(include, True) + compiled = re.compile(regex) + for file in matches: + self.assertIsNotNone(compiled.search(file.encode() if binary else file), file) + for file in misses: + self.assertIsNone(compiled.search(file.encode() if binary else file), file) + + def test_explicit_globstar_contents_shortcut(self): + """ + The explicit **/*/** pattern is not the automatic directory-only */ shortcut. + """ + for pattern_class in (GitIgnoreBasicPattern, GitIgnoreSpecPattern): + pattern = pattern_class('**/*/**') + self.assertIsNone(pattern.match_file('folder/')) + self.assertIsNone(pattern.match_file('file')) + self.assertIsNotNone(pattern.match_file('folder/file')) + self.assertIsNotNone(pattern.match_file('folder/sub/file')) + + def test_regular_directory_and_recursive_shortcuts(self): + """ + Ordinary directory patterns and root-wide recursive shortcuts keep their meaning. + """ + for pattern_class, text in product( + (GitIgnoreBasicPattern, GitIgnoreSpecPattern), ('folder/', '**/', '**/**/'), + ): + with self.subTest(pattern_class=pattern_class, text=text): + pattern = pattern_class(text) + for file in ('folder/', 'folder/file', 'folder/sub/file'): + self.assertIsNotNone(pattern.match_file(file), file) + + def test_newline_directory_names(self): + """ + Recursive directory matching includes newline characters within names. + """ + for pattern_class in (GitIgnoreBasicPattern, GitIgnoreSpecPattern): + pattern = pattern_class('folder/**/') + for file in ('folder/line\nbreak/', 'folder/line\nbreak/file'): + self.assertIsNotNone(pattern.match_file(file), file) + self.assertIsNone(pattern.match_file('folder/line\nbreak')) + + +class GlobstarDirectorySpecTest(GitIgnoreSpecMixin, TestCase): + """ + Tests all matching backend and expression-order configurations. + """ + + def test_literal_prefix(self): + """ + A parent and its immediate files remain available for traversal. + """ + for begin in self.parameterize_from_lines(['folder/**/']): + with begin() as spec: + for file in ('folder/', 'folder/file'): + self.assertFalse(spec.match_file(file), file) + for file in ('folder/sub/', 'folder/sub/file', 'folder/sub/deep/file'): + self.assertTrue(spec.match_file(file), file) + + def test_wildcard_prefix(self): + """ + Wildcard prefixes still require at least one additional directory. + """ + for begin in self.parameterize_from_lines(['*/**/']): + with begin() as spec: + for file in ('folder/', 'folder/file', 'other/file'): + self.assertFalse(spec.match_file(file), file) + for file in ('folder/sub/file', 'other/sub/deep/file'): + self.assertTrue(spec.match_file(file), file)