Skip to content

Commit 8ee67d4

Browse files
committed
pythongh-117151: optimize algorithm to grow the buffer size for readall() on files
1 parent 36ef3bf commit 8ee67d4

1 file changed

Lines changed: 13 additions & 19 deletions

File tree

‎Modules/_io/fileio.c‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,8 @@
4343
# include <windows.h>
4444
#endif
4545

46-
#if BUFSIZ < (8*1024)
47-
# define SMALLCHUNK (8*1024)
48-
#elif (BUFSIZ >= (2 << 25))
49-
# error "unreasonable BUFSIZ > 64 MiB defined"
50-
#else
51-
# define SMALLCHUNK BUFSIZ
52-
#endif
53-
54-
/* Size at which a buffer is considered "large" and behavior should change to
55-
avoid excessive memory allocation */
56-
#define LARGE_BUFFER_CUTOFF_SIZE 65536
46+
#define LARGE_BUFFER_CUTOFF_SIZE (4096*1024)
47+
#define SMALL_BUFFER_SIZE (128*1024)
5748

5849
/*[clinic input]
5950
module _io
@@ -709,16 +700,20 @@ new_buffersize(fileio *self, size_t currentsize)
709700
size_t addend;
710701

711702
/* Expand the buffer by an amount proportional to the current size,
712-
giving us amortized linear-time behavior. For bigger sizes, use a
713-
less-than-double growth factor to avoid excessive allocation. */
703+
giving us amortized linear-time behavior. This heuristic is only used
704+
when the file size was unknown or changed since the file was opened.
705+
For smaller sizes, use exponential growth to avoid many small reads.
706+
For bigger sizes, use a less-than-double growth factor to avoid
707+
excessive allocation.
708+
*/
714709
assert(currentsize <= PY_SSIZE_T_MAX);
715710
if (currentsize > LARGE_BUFFER_CUTOFF_SIZE)
716711
addend = currentsize >> 3;
717712
else
718-
addend = 256 + currentsize;
719-
if (addend < SMALLCHUNK)
713+
addend = 3 * currentsize;
714+
if (addend < SMALL_BUFFER_SIZE)
720715
/* Avoid tiny read() calls. */
721-
addend = SMALLCHUNK;
716+
addend = SMALL_BUFFER_SIZE;
722717
return addend + currentsize;
723718
}
724719

@@ -743,7 +738,6 @@ _io_FileIO_readall_impl(fileio *self)
743738
Py_ssize_t bytes_read = 0;
744739
Py_ssize_t n;
745740
size_t bufsize;
746-
747741
if (self->fd < 0) {
748742
return err_closed();
749743
}
@@ -756,7 +750,7 @@ _io_FileIO_readall_impl(fileio *self)
756750
}
757751
if (end <= 0) {
758752
/* Use a default size and resize as needed. */
759-
bufsize = SMALLCHUNK;
753+
bufsize = SMALL_BUFFER_SIZE;
760754
}
761755
else {
762756
/* This is probably a real file. */
@@ -777,7 +771,7 @@ _io_FileIO_readall_impl(fileio *self)
777771
then calls readall() to get the rest, which would result in allocating
778772
more than required. Guard against that for larger files where we expect
779773
the I/O time to dominate anyways while keeping small files fast. */
780-
if (bufsize > LARGE_BUFFER_CUTOFF_SIZE) {
774+
if (bufsize > SMALL_BUFFER_SIZE) {
781775
Py_BEGIN_ALLOW_THREADS
782776
_Py_BEGIN_SUPPRESS_IPH
783777
#ifdef MS_WINDOWS

0 commit comments

Comments
 (0)