4343# include <windows.h>
4444#endif
4545
46- #if BUFSIZ < (8 * 1024 )
47- # define SMALLCHUNK (8*1024)
48- #elif (BUFSIZ >= (2 << 25 ))
49- # error "unreasonable BUFSIZ > 64 MiB defined"
50- #else
51- # define SMALLCHUNK BUFSIZ
52- #endif
53-
54- /* Size at which a buffer is considered "large" and behavior should change to
55- avoid excessive memory allocation */
56- #define LARGE_BUFFER_CUTOFF_SIZE 65536
46+ #define LARGE_BUFFER_CUTOFF_SIZE (4096*1024)
47+ #define SMALL_BUFFER_SIZE (128*1024)
5748
5849/*[clinic input]
5950module _io
@@ -709,16 +700,20 @@ new_buffersize(fileio *self, size_t currentsize)
709700 size_t addend ;
710701
711702 /* Expand the buffer by an amount proportional to the current size,
712- giving us amortized linear-time behavior. For bigger sizes, use a
713- less-than-double growth factor to avoid excessive allocation. */
703+ giving us amortized linear-time behavior. This heuristic is only used
704+ when the file size was unknown or changed since the file was opened.
705+ For smaller sizes, use exponential growth to avoid many small reads.
706+ For bigger sizes, use a less-than-double growth factor to avoid
707+ excessive allocation.
708+ */
714709 assert (currentsize <= PY_SSIZE_T_MAX );
715710 if (currentsize > LARGE_BUFFER_CUTOFF_SIZE )
716711 addend = currentsize >> 3 ;
717712 else
718- addend = 256 + currentsize ;
719- if (addend < SMALLCHUNK )
713+ addend = 3 * currentsize ;
714+ if (addend < SMALL_BUFFER_SIZE )
720715 /* Avoid tiny read() calls. */
721- addend = SMALLCHUNK ;
716+ addend = SMALL_BUFFER_SIZE ;
722717 return addend + currentsize ;
723718}
724719
@@ -743,7 +738,6 @@ _io_FileIO_readall_impl(fileio *self)
743738 Py_ssize_t bytes_read = 0 ;
744739 Py_ssize_t n ;
745740 size_t bufsize ;
746-
747741 if (self -> fd < 0 ) {
748742 return err_closed ();
749743 }
@@ -756,7 +750,7 @@ _io_FileIO_readall_impl(fileio *self)
756750 }
757751 if (end <= 0 ) {
758752 /* Use a default size and resize as needed. */
759- bufsize = SMALLCHUNK ;
753+ bufsize = SMALL_BUFFER_SIZE ;
760754 }
761755 else {
762756 /* This is probably a real file. */
@@ -777,7 +771,7 @@ _io_FileIO_readall_impl(fileio *self)
777771 then calls readall() to get the rest, which would result in allocating
778772 more than required. Guard against that for larger files where we expect
779773 the I/O time to dominate anyways while keeping small files fast. */
780- if (bufsize > LARGE_BUFFER_CUTOFF_SIZE ) {
774+ if (bufsize > SMALL_BUFFER_SIZE ) {
781775 Py_BEGIN_ALLOW_THREADS
782776 _Py_BEGIN_SUPPRESS_IPH
783777#ifdef MS_WINDOWS
0 commit comments