Skip to content

Commit 74a495c

Browse files
committed
pythonGH-119169: Speed up os.walk()
Handle errors from `os.scandir()` and `ScandirIterator` similarly, which lets us loop over directory entries with `for`. In top-down mode, call `os.path.join()` at most once per iteration.
1 parent 3c28510 commit 74a495c

2 files changed

Lines changed: 31 additions & 41 deletions

File tree

‎Lib/os.py‎

Lines changed: 29 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -361,39 +361,23 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
361361
# minor reason when (say) a thousand readable directories are still
362362
# left to visit.
363363
try:
364-
scandir_it = scandir(top)
365-
except OSError as error:
366-
if onerror is not None:
367-
onerror(error)
368-
continue
369-
370-
cont = False
371-
with scandir_it:
372-
while True:
373-
try:
364+
with scandir(top) as entries:
365+
for entry in entries:
374366
try:
375-
entry = next(scandir_it)
376-
except StopIteration:
377-
break
378-
except OSError as error:
379-
if onerror is not None:
380-
onerror(error)
381-
cont = True
382-
break
383-
384-
try:
385-
is_dir = entry.is_dir()
386-
except OSError:
387-
# If is_dir() raises an OSError, consider the entry not to
388-
# be a directory, same behaviour as os.path.isdir().
389-
is_dir = False
390-
391-
if is_dir:
392-
dirs.append(entry.name)
393-
else:
394-
nondirs.append(entry.name)
367+
is_dir = entry.is_dir()
368+
except OSError:
369+
# If is_dir() raises an OSError, consider the entry not to
370+
# be a directory, same behaviour as os.path.isdir().
371+
is_dir = False
372+
373+
if is_dir:
374+
dirs.append(entry.name)
375+
else:
376+
nondirs.append(entry.name)
377+
continue
378+
if topdown:
379+
continue
395380

396-
if not topdown and is_dir:
397381
# Bottom-up: traverse into sub-directory, but exclude
398382
# symlinks to directories if followlinks is False
399383
if followlinks:
@@ -410,21 +394,25 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
410394

411395
if walk_into:
412396
walk_dirs.append(entry.path)
413-
if cont:
397+
except OSError as error:
398+
if onerror is not None:
399+
onerror(error)
414400
continue
415401

416402
if topdown:
417403
# Yield before sub-directory traversal if going top down
418404
yield top, dirs, nondirs
419-
# Traverse into sub-directories
420-
for dirname in reversed(dirs):
421-
new_path = join(top, dirname)
422-
# bpo-23605: os.path.islink() is used instead of caching
423-
# entry.is_symlink() result during the loop on os.scandir() because
424-
# the caller can replace the directory entry during the "yield"
425-
# above.
426-
if followlinks or not islink(new_path):
427-
stack.append(new_path)
405+
if dirs:
406+
# Traverse into sub-directories
407+
prefix = join(top, top[:0])
408+
for dirname in reversed(dirs):
409+
new_path = prefix + dirname
410+
# bpo-23605: os.path.islink() is used instead of caching
411+
# entry.is_symlink() result during the loop on os.scandir() because
412+
# the caller can replace the directory entry during the "yield"
413+
# above.
414+
if followlinks or not islink(new_path):
415+
stack.append(new_path)
428416
else:
429417
# Yield after sub-directory traversal if going bottom up
430418
stack.append((top, dirs, nondirs))
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Slightly speed up :func:`os.walk` by simplifying exception handling and
2+
reducing path joining.

0 commit comments

Comments
 (0)