|
| 1 | +From 7a2b7388233ea0c647168c5792dc31f3051fe177 Mon Sep 17 00:00:00 2001 |
| 2 | +From: Petr Viktorin <encukou@gmail.com> |
| 3 | +Date: Mon, 31 Aug 2026 21:04:04 +0200 |
| 4 | +Subject: [PATCH 1/3] [3.15] gh-156002: Bound zipfile decompression for |
| 5 | + bzip2/LZMA/Zstandard (GH-156003) (GH-156362) |
| 6 | + |
| 7 | +Patch by @tonghuaroot. |
| 8 | + |
| 9 | +zipfile.ZipExtFile._read1() bounds the output of each decompress() call |
| 10 | +for DEFLATE members by passing a max_length to zlib, but for bzip2, LZMA, |
| 11 | +and Zstandard members it called decompress() with no bound. A whole |
| 12 | +compressed chunk was therefore expanded into a single allocation before |
| 13 | +the data[:self._left] clip ran, so a consumer that deliberately reads in |
| 14 | +small chunks to limit memory (for example zf.open(name).read(8192)) was |
| 15 | +silently unprotected for non-DEFLATE members. A small, spec-conformant |
| 16 | +archive member declaring a large uncompressed size could drive multi-GB |
| 17 | +peak memory. |
| 18 | + |
| 19 | +_read1() now passes a per-call bound to the non-DEFLATE decompress() |
| 20 | +(mirroring the DEFLATE branch) and drains the decompressor's internal |
| 21 | +buffer across calls by checking needs_input before reading more |
| 22 | +compressed input. zipfile's LZMADecompressor wrapper forwards max_length |
| 23 | +and exposes needs_input so the bound also holds for LZMA members. |
| 24 | + |
| 25 | +(cherry picked from commit f897dbf2f36a5935700b7c2d94d4681d2136b7d4) |
| 26 | +(cherry picked from commit 1b424c0178a01e155fd0267dc28a8fc1159b33a8) |
| 27 | + |
| 28 | +Co-authored-by: Petr Viktorin <encukou@gmail.com> |
| 29 | +Co-authored-by: tonghuaroot <tonghuaroot@gmail.com> |
| 30 | +--- |
| 31 | + Lib/test/test_zipfile/test_core.py | 104 ++++++++++ |
| 32 | + Lib/zipfile/__init__.py | 35 ++- |
| 33 | + Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst | 5 |
| 34 | + Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst | 4 |
| 35 | + 4 files changed, 143 insertions(+), 5 deletions(-) |
| 36 | + create mode 100644 Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst |
| 37 | + |
| 38 | +Index: Python-3.12.14/Lib/test/test_zipfile/test_core.py |
| 39 | +=================================================================== |
| 40 | +--- Python-3.12.14.orig/Lib/test/test_zipfile/test_core.py 2026-09-18 02:13:30.182873716 +0200 |
| 41 | ++++ Python-3.12.14/Lib/test/test_zipfile/test_core.py 2026-09-18 02:13:32.565920004 +0200 |
| 42 | +@@ -2590,6 +2590,110 @@ |
| 43 | + unlink(TESTFN2) |
| 44 | + |
| 45 | + |
| 46 | ++class AbstractBoundedDecompressTests: |
| 47 | ++ # ZipExtFile._read1() bounds the output of each decompress() call so that a |
| 48 | ++ # small member declaring a large uncompressed size cannot expand into one |
| 49 | ++ # unbounded read. |
| 50 | ++ def test_read1_output_is_bounded(self): |
| 51 | ++ buf = io.BytesIO() |
| 52 | ++ with zipfile.ZipFile(buf, "w", compression=self.compression) as zf: |
| 53 | ++ zf.writestr("big", b"\0" * (4 * 1024 * 1024)) |
| 54 | ++ with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: |
| 55 | ++ with zf.open("big") as f: |
| 56 | ++ self.assertLessEqual(len(f._read1(100)), f.MIN_READ_SIZE) |
| 57 | ++ |
| 58 | ++ |
| 59 | ++class StoredBoundedDecompressTests(AbstractBoundedDecompressTests, |
| 60 | ++ unittest.TestCase): |
| 61 | ++ compression = zipfile.ZIP_STORED |
| 62 | ++ |
| 63 | ++ |
| 64 | ++@requires_zlib() |
| 65 | ++class DeflateBoundedDecompressTests(AbstractBoundedDecompressTests, |
| 66 | ++ unittest.TestCase): |
| 67 | ++ compression = zipfile.ZIP_DEFLATED |
| 68 | ++ |
| 69 | ++ |
| 70 | ++@requires_bz2() |
| 71 | ++class Bzip2BoundedDecompressTests(AbstractBoundedDecompressTests, |
| 72 | ++ unittest.TestCase): |
| 73 | ++ compression = zipfile.ZIP_BZIP2 |
| 74 | ++ |
| 75 | ++ |
| 76 | ++@requires_lzma() |
| 77 | ++class LzmaBoundedDecompressTests(AbstractBoundedDecompressTests, |
| 78 | ++ unittest.TestCase): |
| 79 | ++ compression = zipfile.ZIP_LZMA |
| 80 | ++ |
| 81 | ++ |
| 82 | ++ |
| 83 | ++class MonkeypatchedDecompressorTests(unittest.TestCase): |
| 84 | ++ # Some third-party projects monkey-patch _get_decompressor() to add |
| 85 | ++ # additional compression schemes. This can break at any time as the |
| 86 | ++ # internal compressor objects change. |
| 87 | ++ # To protect users, we try to keep this case working. |
| 88 | ++ # See also: GH-156002 and GH-113767. |
| 89 | ++ COMPRESSION = 99 |
| 90 | ++ |
| 91 | ++ class Compressor: |
| 92 | ++ """Compressor with only the original BZ2Compressor API""" |
| 93 | ++ def compress(self, data): |
| 94 | ++ return data.swapcase() |
| 95 | ++ |
| 96 | ++ def flush(self): |
| 97 | ++ return b'' |
| 98 | ++ |
| 99 | ++ class Decompressor: |
| 100 | ++ """Decompressor with only the 3.3+ BZ2Decompressor API""" |
| 101 | ++ eof = False |
| 102 | ++ |
| 103 | ++ def decompress(self, data): |
| 104 | ++ return data.swapcase() |
| 105 | ++ |
| 106 | ++ def setUp(self): |
| 107 | ++ orig_check_compression = zipfile._check_compression |
| 108 | ++ orig_get_compressor = zipfile._get_compressor |
| 109 | ++ orig_get_decompressor = zipfile._get_decompressor |
| 110 | ++ |
| 111 | ++ def check_compression(compression): |
| 112 | ++ if compression != self.COMPRESSION: |
| 113 | ++ orig_check_compression(compression) |
| 114 | ++ |
| 115 | ++ def get_compressor(compress_type, compresslevel=None): |
| 116 | ++ if compress_type == self.COMPRESSION: |
| 117 | ++ return self.Compressor() |
| 118 | ++ return orig_get_compressor(compress_type, compresslevel) |
| 119 | ++ |
| 120 | ++ def get_decompressor(compress_type): |
| 121 | ++ if compress_type == self.COMPRESSION: |
| 122 | ++ return self.Decompressor() |
| 123 | ++ return orig_get_decompressor(compress_type) |
| 124 | ++ |
| 125 | ++ self.enterContext(mock.patch.object( |
| 126 | ++ zipfile, '_check_compression', check_compression)) |
| 127 | ++ self.enterContext(mock.patch.object( |
| 128 | ++ zipfile, '_get_compressor', get_compressor)) |
| 129 | ++ self.enterContext(mock.patch.object( |
| 130 | ++ zipfile, '_get_decompressor', get_decompressor)) |
| 131 | ++ |
| 132 | ++ def test_roundtrip_monkeypatched_decompressor(self): |
| 133 | ++ data = bytes(range(256)) * 8 |
| 134 | ++ buf = io.BytesIO() |
| 135 | ++ with zipfile.ZipFile(buf, "w", compression=self.COMPRESSION) as zf: |
| 136 | ++ zf.writestr("member", data) |
| 137 | ++ self.assertIn(data.swapcase(), buf.getvalue()) |
| 138 | ++ with zipfile.ZipFile(io.BytesIO(buf.getvalue())) as zf: |
| 139 | ++ self.assertEqual(zf.read("member"), data) |
| 140 | ++ with zf.open("member") as f: |
| 141 | ++ self.assertEqual(f.read(100), data[:100]) |
| 142 | ++ self.assertEqual(f.read1(100), data[100:200]) |
| 143 | ++ f.seek(-100, os.SEEK_END) |
| 144 | ++ self.assertEqual(f.read(), data[-100:]) |
| 145 | ++ # Rewinding past the read buffer re-creates the decompressor. |
| 146 | ++ f.seek(0) |
| 147 | ++ self.assertEqual(f.read(), data) |
| 148 | ++ |
| 149 | ++ |
| 150 | + class AbstractBadCrcTests: |
| 151 | + def test_testzip_with_bad_crc(self): |
| 152 | + """Tests that files with bad CRCs return their name from testzip.""" |
| 153 | +Index: Python-3.12.14/Lib/zipfile/__init__.py |
| 154 | +=================================================================== |
| 155 | +--- Python-3.12.14.orig/Lib/zipfile/__init__.py 2026-09-18 02:13:30.518309790 +0200 |
| 156 | ++++ Python-3.12.14/Lib/zipfile/__init__.py 2026-09-18 02:13:32.567802775 +0200 |
| 157 | +@@ -698,7 +698,16 @@ |
| 158 | + self._unconsumed = b'' |
| 159 | + self.eof = False |
| 160 | + |
| 161 | +- def decompress(self, data): |
| 162 | ++ @property |
| 163 | ++ def needs_input(self): |
| 164 | ++ # While the LZMA properties header is still being buffered, more input |
| 165 | ++ # is required; afterwards defer to the wrapped decompressor so a bounded |
| 166 | ++ # decompress() call can be drained across reads. |
| 167 | ++ if self._decomp is None: |
| 168 | ++ return True |
| 169 | ++ return self._decomp.needs_input |
| 170 | ++ |
| 171 | ++ def decompress(self, data, max_length=-1): |
| 172 | + if self._decomp is None: |
| 173 | + self._unconsumed += data |
| 174 | + if len(self._unconsumed) <= 4: |
| 175 | +@@ -714,7 +723,7 @@ |
| 176 | + data = self._unconsumed[4 + psize:] |
| 177 | + del self._unconsumed |
| 178 | + |
| 179 | +- result = self._decomp.decompress(data) |
| 180 | ++ result = self._decomp.decompress(data, max_length) |
| 181 | + self.eof = self._decomp.eof |
| 182 | + return result |
| 183 | + |
| 184 | +@@ -1074,8 +1083,15 @@ |
| 185 | + data = self._decompressor.unconsumed_tail |
| 186 | + if n > len(data): |
| 187 | + data += self._read2(n - len(data)) |
| 188 | +- else: |
| 189 | ++ elif self._compress_type == ZIP_STORED: |
| 190 | + data = self._read2(n) |
| 191 | ++ else: |
| 192 | ++ # bzip2/lzma/zstd: a bounded decompress() call may leave input |
| 193 | ++ # buffered inside the decompressor; drain that before reading more. |
| 194 | ++ if getattr(self._decompressor, "needs_input", True): |
| 195 | ++ data = self._read2(n) |
| 196 | ++ else: |
| 197 | ++ data = b'' |
| 198 | + |
| 199 | + if self._compress_type == ZIP_STORED: |
| 200 | + self._eof = self._compress_left <= 0 |
| 201 | +@@ -1088,8 +1104,17 @@ |
| 202 | + if self._eof: |
| 203 | + data += self._decompressor.flush() |
| 204 | + else: |
| 205 | +- data = self._decompressor.decompress(data) |
| 206 | +- self._eof = self._decompressor.eof or self._compress_left <= 0 |
| 207 | ++ # Bound the output of a single decompress() call (mirroring the |
| 208 | ++ # DEFLATE path above) so that a small compressed member cannot |
| 209 | ++ # expand into one unbounded read. |
| 210 | ++ try: |
| 211 | ++ data = self._decompressor.decompress(data, max(n, self.MIN_READ_SIZE)) |
| 212 | ++ except TypeError: |
| 213 | ++ # See MonkeypatchedDecompressorTests in test_core.py |
| 214 | ++ data = self._decompressor.decompress(data) |
| 215 | ++ self._eof = (self._decompressor.eof or |
| 216 | ++ self._compress_left <= 0 and |
| 217 | ++ getattr(self._decompressor, "needs_input", True)) |
| 218 | + |
| 219 | + data = data[:self._left] |
| 220 | + self._left -= len(data) |
| 221 | +Index: Python-3.12.14/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst |
| 222 | +=================================================================== |
| 223 | +--- /dev/null 1970-01-01 00:00:00.000000000 +0000 |
| 224 | ++++ Python-3.12.14/Misc/NEWS.d/next/Library/2026-09-08-13-06-29.gh-issue-156002.vmOC8T.rst 2026-09-18 02:13:32.568427063 +0200 |
| 225 | +@@ -0,0 +1,5 @@ |
| 226 | ++:mod:`zipfile` again reads members through a third-party decompressor |
| 227 | ++installed by monkey-patching the private ``_get_decompressor()`` to return an |
| 228 | ++object that only implements old BZ2Decompressor API from Python 3.3. |
| 229 | ++Note that decompressors without ``needs_input`` and two-argument |
| 230 | ++``decompress()`` are vulnerable to :cve:`2026-15310`. |
| 231 | +Index: Python-3.12.14/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst |
| 232 | +=================================================================== |
| 233 | +--- /dev/null 1970-01-01 00:00:00.000000000 +0000 |
| 234 | ++++ Python-3.12.14/Misc/NEWS.d/next/Security/2026-08-18-13-54-05.gh-issue-156002.CcWXPP.rst 2026-09-18 02:13:32.568144339 +0200 |
| 235 | +@@ -0,0 +1,4 @@ |
| 236 | ++Bound the amount of data :mod:`zipfile` decompresses per read for members |
| 237 | ++compressed with bzip2, LZMA, or Zstandard, matching the existing limit for |
| 238 | ++deflate. A small archive member could previously expand into an unbounded |
| 239 | ++allocation even when read in small chunks. |
0 commit comments