Skip to content

Commit 25dded0

Browse files
kristjanvalurkristjanvalur
committed
Make the various iterators' "setstate" sliently and consistently clip the
index. This avoids the possibility of setting an iterator to an invalid state.
1 parent 4ca688e commit 25dded0

8 files changed

Lines changed: 66 additions & 15 deletions

File tree

‎Lib/test/test_range.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,18 @@ def test_iterator_pickling(self):
380380
self.assertEqual(list(it), data[1:])
381381

382382
def test_exhausted_iterator_pickling(self):
383+
r = range(2**65, 2**65+2)
384+
i = iter(r)
385+
while True:
386+
r = next(i)
387+
if r == 2**65+1:
388+
break
389+
d = pickle.dumps(i)
390+
i2 = pickle.loads(d)
391+
self.assertEqual(list(i), [])
392+
self.assertEqual(list(i2), [])
393+
394+
def test_large_exhausted_iterator_pickling(self):
383395
r = range(20)
384396
i = iter(r)
385397
while True:

‎Modules/arraymodule.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2798,6 +2798,8 @@ arrayiter_setstate(arrayiterobject *it, PyObject *state)
27982798
return NULL;
27992799
if (index < 0)
28002800
index = 0;
2801+
else if (index > Py_SIZE(it->ao))
2802+
index = Py_SIZE(it->ao); /* iterator exhausted */
28012803
it->index = index;
28022804
Py_RETURN_NONE;
28032805
}

‎Objects/bytearrayobject.c‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3043,9 +3043,13 @@ bytearrayiter_setstate(bytesiterobject *it, PyObject *state)
30433043
Py_ssize_t index = PyLong_AsSsize_t(state);
30443044
if (index == -1 && PyErr_Occurred())
30453045
return NULL;
3046-
if (index < 0)
3047-
index = 0;
3048-
it->it_index = index;
3046+
if (it->it_seq != NULL) {
3047+
if (index < 0)
3048+
index = 0;
3049+
else if (index > PyByteArray_GET_SIZE(it->it_seq))
3050+
index = PyByteArray_GET_SIZE(it->it_seq); /* iterator exhausted */
3051+
it->it_index = index;
3052+
}
30493053
Py_RETURN_NONE;
30503054
}
30513055

‎Objects/bytesobject.c‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2996,9 +2996,13 @@ striter_setstate(striterobject *it, PyObject *state)
29962996
Py_ssize_t index = PyLong_AsSsize_t(state);
29972997
if (index == -1 && PyErr_Occurred())
29982998
return NULL;
2999-
if (index < 0)
3000-
index = 0;
3001-
it->it_index = index;
2999+
if (it->it_seq != NULL) {
3000+
if (index < 0)
3001+
index = 0;
3002+
else if (index > PyBytes_GET_SIZE(it->it_seq))
3003+
index = PyBytes_GET_SIZE(it->it_seq); /* iterator exhausted */
3004+
it->it_index = index;
3005+
}
30023006
Py_RETURN_NONE;
30033007
}
30043008

‎Objects/listobject.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2803,6 +2803,8 @@ listiter_setstate(listiterobject *it, PyObject *state)
28032803
if (it->it_seq != NULL) {
28042804
if (index < 0)
28052805
index = 0;
2806+
else if (index > PyList_GET_SIZE(it->it_seq))
2807+
index = PyList_GET_SIZE(it->it_seq); /* iterator exhausted */
28062808
it->it_index = index;
28072809
}
28082810
Py_RETURN_NONE;

‎Objects/rangeobject.c‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,10 +1000,11 @@ rangeiter_setstate(rangeiterobject *r, PyObject *state)
10001000
long index = PyLong_AsLong(state);
10011001
if (index == -1 && PyErr_Occurred())
10021002
return NULL;
1003-
if (index < 0 || index > r->len) {
1004-
PyErr_SetString(PyExc_ValueError, "index out of range");
1005-
return NULL;
1006-
}
1003+
/* silently clip the index value */
1004+
if (index < 0)
1005+
index = 0;
1006+
else if (index > r->len)
1007+
index = r->len; /* exhausted iterator */
10071008
r->index = index;
10081009
Py_RETURN_NONE;
10091010
}
@@ -1178,6 +1179,28 @@ longrangeiter_reduce(longrangeiterobject *r)
11781179
static PyObject *
11791180
longrangeiter_setstate(longrangeiterobject *r, PyObject *state)
11801181
{
1182+
int cmp;
1183+
1184+
/* clip the value */
1185+
PyObject *zero = PyLong_FromLong(0);
1186+
if (zero == NULL)
1187+
return NULL;
1188+
cmp = PyObject_RichCompareBool(state, zero, Py_LT);
1189+
if (cmp > 0) {
1190+
Py_CLEAR(r->index);
1191+
r->index = zero;
1192+
Py_RETURN_NONE;
1193+
}
1194+
Py_DECREF(zero);
1195+
if (cmp < 0)
1196+
return NULL;
1197+
1198+
cmp = PyObject_RichCompareBool(r->len, state, Py_LT);
1199+
if (cmp < 0)
1200+
return NULL;
1201+
if (cmp > 0)
1202+
state = r->len;
1203+
11811204
Py_CLEAR(r->index);
11821205
r->index = state;
11831206
Py_INCREF(r->index);

‎Objects/tupleobject.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -998,8 +998,8 @@ tupleiter_setstate(tupleiterobject *it, PyObject *state)
998998
if (it->it_seq != NULL) {
999999
if (index < 0)
10001000
index = 0;
1001-
else if (it->it_seq != NULL && index > PyTuple_GET_SIZE(it->it_seq))
1002-
index = PyTuple_GET_SIZE(it->it_seq);
1001+
else if (index > PyTuple_GET_SIZE(it->it_seq))
1002+
index = PyTuple_GET_SIZE(it->it_seq); /* exhausted iterator */
10031003
it->it_index = index;
10041004
}
10051005
Py_RETURN_NONE;

‎Objects/unicodeobject.c‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14420,9 +14420,13 @@ unicodeiter_setstate(unicodeiterobject *it, PyObject *state)
1442014420
Py_ssize_t index = PyLong_AsSsize_t(state);
1442114421
if (index == -1 && PyErr_Occurred())
1442214422
return NULL;
14423-
if (index < 0)
14424-
index = 0;
14425-
it->it_index = index;
14423+
if (it->it_seq != NULL) {
14424+
if (index < 0)
14425+
index = 0;
14426+
else if (index > PyUnicode_GET_LENGTH(it->it_seq))
14427+
index = PyUnicode_GET_LENGTH(it->it_seq); /* iterator truncated */
14428+
it->it_index = index;
14429+
}
1442614430
Py_RETURN_NONE;
1442714431
}
1442814432

0 commit comments

Comments
 (0)