Skip to content

Commit e3d0923

Browse files
committed
pythonGH-102613: Improve performance of pathlib.Path.rglob()
Stop de-duplicating results in `_RecursiveWildcardSelector`. A new `_DoubleRecursiveWildcardSelector` class is introduced which performs de-duplication, but this is used _only_ for patterns with multiple non-adjacent `**` segments, such as `path.glob('**/foo/**')`. By avoiding the use of a set, `PurePath.__hash__()` is not called, and so paths do not need to be parsed and (case-) normalised. Also merge adjacent '**' segments in patterns.
1 parent de7f694 commit e3d0923

2 files changed

Lines changed: 31 additions & 11 deletions

File tree

‎Lib/pathlib.py‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,12 @@ def _make_selector(pattern_parts, flavour, case_sensitive):
6868
if not pat:
6969
return _TerminatingSelector()
7070
if pat == '**':
71-
cls = _RecursiveWildcardSelector
71+
while child_parts and child_parts[0] == '**':
72+
child_parts = child_parts[1:]
73+
if '**' in child_parts:
74+
cls = _DoubleRecursiveWildcardSelector
75+
else:
76+
cls = _RecursiveWildcardSelector
7277
elif pat == '..':
7378
cls = _ParentSelector
7479
elif '**' in pat:
@@ -183,20 +188,32 @@ def _iterate_directories(self, parent_path, scandir):
183188

184189
def _select_from(self, parent_path, scandir):
185190
try:
186-
yielded = set()
187-
try:
188-
successor_select = self.successor._select_from
189-
for starting_point in self._iterate_directories(parent_path, scandir):
190-
for p in successor_select(starting_point, scandir):
191-
if p not in yielded:
192-
yield p
193-
yielded.add(p)
194-
finally:
195-
yielded.clear()
191+
successor_select = self.successor._select_from
192+
for starting_point in self._iterate_directories(parent_path, scandir):
193+
for p in successor_select(starting_point, scandir):
194+
yield p
196195
except PermissionError:
197196
return
198197

199198

199+
class _DoubleRecursiveWildcardSelector(_RecursiveWildcardSelector):
200+
"""
201+
Like _RecursiveWildcardSelector, but also de-duplicates results from
202+
successive selectors. This is necessary if the pattern contains
203+
multiple non-adjacent '**' segments.
204+
"""
205+
206+
def _select_from(self, parent_path, scandir):
207+
yielded = set()
208+
try:
209+
for p in super()._select_from(parent_path, scandir):
210+
if p not in yielded:
211+
yield p
212+
yielded.add(p)
213+
finally:
214+
yielded.clear()
215+
216+
200217
#
201218
# Public API
202219
#
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Improve performance of :meth:`pathlib.Path.glob` when expanding recursive
2+
wildcards ("``**``") by merging adjacent wildcards and de-duplicating
3+
results only when necessary.

0 commit comments

Comments
 (0)