Skip to content

Commit 1bb8e19

Browse files
edgarrmondragonedgarrmondragon
authored
Merge branch 'master' into stubgen-sentinel
2 parents 4adf0b0 + 0861bb6 commit 1bb8e19

18 files changed

Lines changed: 379 additions & 45 deletions

‎mypy/build.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,12 +288,13 @@ def __init__(self, status_file: str, options_data: str, env: Mapping[str, str])
288288

289289
def connect(self) -> None:
290290
end_time = time.time() + WORKER_START_TIMEOUT
291-
last_exception: Exception | None = None
292-
while time.time() < end_time:
291+
while True:
293292
try:
294293
data = read_status(self.status_file)
295294
except BadStatus as exc:
296295
last_exception = exc
296+
if time.time() >= end_time:
297+
break
297298
time.sleep(WORKER_START_INTERVAL)
298299
continue
299300
try:

‎mypy/checkexpr.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3311,7 +3311,7 @@ def combine_function_signatures(self, types: list[ProperType]) -> AnyType | Call
33113311
return AnyType(TypeOfAny.special_form)
33123312
callables = cast("list[CallableType]", types)
33133313

3314-
combined = union_function_signatures(callables)
3314+
combined = union_function_signatures(callables, simplify_unions=True)
33153315
if combined is not None:
33163316
return combined
33173317

‎mypy/constraints.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
TypeInfo,
2323
)
2424
from mypy.types import (
25+
MAX_PROTOCOL_DEPTH,
2526
TUPLE_LIKE_INSTANCE_NAMES,
2627
AnyType,
2728
CallableType,
@@ -333,9 +334,9 @@ def _infer_constraints(
333334
# Type inference shouldn't be affected by whether union types have been simplified.
334335
# We however keep any ErasedType items, so that the caller will see it when using
335336
# checkexpr.has_erased_component().
336-
if isinstance(template, UnionType):
337+
if not type_state.keep_unions and isinstance(template, UnionType):
337338
template = mypy.typeops.make_simplified_union(template.items, keep_erased=True)
338-
if isinstance(actual, UnionType):
339+
if not type_state.keep_unions and isinstance(actual, UnionType):
339340
actual = mypy.typeops.make_simplified_union(actual.items, keep_erased=True)
340341

341342
# Ignore Any types from the type suggestion engine to avoid them
@@ -750,7 +751,9 @@ def visit_instance(self, template: Instance) -> list[Constraint]:
750751
if isinstance(actual, (CallableType, Overloaded)) and template.type.is_protocol:
751752
if "__call__" in template.type.protocol_members:
752753
# Special case: a generic callback protocol
753-
if not any(template == t for t in template.type.inferring):
754+
if len(template.type.inferring) < MAX_PROTOCOL_DEPTH and not any(
755+
template == t for t in template.type.inferring
756+
):
754757
template.type.inferring.append(template)
755758
call = mypy.subtypes.find_member(
756759
"__call__", template, actual, is_operator=True
@@ -944,6 +947,7 @@ def visit_instance(self, template: Instance) -> list[Constraint]:
944947
if (
945948
template.type.is_protocol
946949
and self.direction == SUPERTYPE_OF
950+
and len(template.type.inferring) < MAX_PROTOCOL_DEPTH
947951
and
948952
# We avoid infinite recursion for structural subtypes by checking
949953
# whether this type already appeared in the inference chain.
@@ -967,6 +971,7 @@ def visit_instance(self, template: Instance) -> list[Constraint]:
967971
elif (
968972
instance.type.is_protocol
969973
and self.direction == SUBTYPE_OF
974+
and len(instance.type.inferring) < MAX_PROTOCOL_DEPTH
970975
and
971976
# We avoid infinite recursion for structural subtypes also here.
972977
not any(instance == i for i in reversed(instance.type.inferring))
@@ -1013,6 +1018,7 @@ def visit_instance(self, template: Instance) -> list[Constraint]:
10131018
if (
10141019
template.type.is_protocol
10151020
and self.direction == SUPERTYPE_OF
1021+
and len(template.type.inferring) < MAX_PROTOCOL_DEPTH
10161022
and not any(template == t for t in reversed(template.type.inferring))
10171023
and mypy.subtypes.is_protocol_implementation(instance, erased, skip=["__call__"])
10181024
):

‎mypy/defaults.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
WORKER_START_TIMEOUT: Final = 10 if slow_fs else 3
6161
WORKER_SHUTDOWN_TIMEOUT: Final = 3 if sys.platform != "win32" else 10
6262

63-
WORKER_CONNECTION_TIMEOUT: Final = 10
63+
# On macOS some (binary) mypy plugins can take a lot of time to import
64+
# because of XProtect scanning.
65+
WORKER_CONNECTION_TIMEOUT: Final = 30 if sys.platform == "darwin" else 10
6466
WORKER_IDLE_TIMEOUT: Final = 600
6567
WORKER_DONE_TIMEOUT: Final = 600

‎mypy/ipc.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ class IPCBase:
6666
def __init__(self, name: str, timeout: float | None) -> None:
6767
self.name = name
6868
self.timeout = timeout
69+
# Windows-specific I/O timeout overrides should be applied here.
70+
self.win_io_timeout = timeout
6971
self.message_size: int | None = None
7072
self.buffer = bytearray()
7173

@@ -101,7 +103,11 @@ def read_bytes(self, size: int = MAX_READ) -> bytes:
101103
ov, err = _winapi.ReadFile(self.connection, size, overlapped=True)
102104
try:
103105
if err == _winapi.ERROR_IO_PENDING:
104-
timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE
106+
timeout = (
107+
int(self.win_io_timeout * 1000)
108+
if self.win_io_timeout
109+
else _winapi.INFINITE
110+
)
105111
res = _winapi.WaitForSingleObject(ov.event, timeout)
106112
if res != _winapi.WAIT_OBJECT_0:
107113
raise IPCException(f"Bad result from I/O wait: {res}")
@@ -158,7 +164,11 @@ def write_bytes(self, data: bytes) -> None:
158164
ov, err = _winapi.WriteFile(self.connection, encoded_data, overlapped=True)
159165
try:
160166
if err == _winapi.ERROR_IO_PENDING:
161-
timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE
167+
timeout = (
168+
int(self.win_io_timeout * 1000)
169+
if self.win_io_timeout
170+
else _winapi.INFINITE
171+
)
162172
res = _winapi.WaitForSingleObject(ov.event, timeout)
163173
if res != _winapi.WAIT_OBJECT_0:
164174
raise IPCException(f"Bad result from I/O wait: {res}")
@@ -248,6 +258,11 @@ def __init__(self, name: str, timeout: float | None = None) -> None:
248258
name = f"{name}.sock"
249259
super().__init__(name, timeout)
250260
if sys.platform == "win32":
261+
# Unlike the client, a server applies its timeout only to accepting a
262+
# connection, never to the traffic that follows: see __enter__() below.
263+
# On POSIX this happens naturally after the sock.accept() call. On
264+
# Windows we need to set this manually to ensure equivalent behavior.
265+
self.win_io_timeout = None
251266
self.connection = _winapi.CreateNamedPipe(
252267
self.name,
253268
_winapi.PIPE_ACCESS_DUPLEX

‎mypy/subtypes.py‎

Lines changed: 54 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
from mypy.options import Options
3838
from mypy.state import state
3939
from mypy.types import (
40+
MAX_PROTOCOL_DEPTH,
4041
MYPYC_NATIVE_INT_NAMES,
4142
TUPLE_LIKE_INSTANCE_NAMES,
4243
TYPED_NAMEDTUPLE_NAMES,
@@ -740,17 +741,16 @@ def visit_callable_type(self, left: CallableType) -> bool:
740741
# Similarly, if one function has `TypeIs` and the other does not,
741742
# they are not compatible.
742743
return False
744+
strict_concatenate = False
745+
if options := self.options:
746+
strict_concatenate = options.extra_checks or options.strict_concatenate
743747
return is_callable_compatible(
744748
left,
745749
right,
746750
is_compat=self._is_subtype,
747751
is_proper_subtype=self.proper_subtype,
748752
ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names,
749-
strict_concatenate=(
750-
(self.options.extra_checks or self.options.strict_concatenate)
751-
if self.options
752-
else False
753-
),
753+
strict_concatenate=strict_concatenate,
754754
)
755755
elif isinstance(right, Overloaded):
756756
return all(self._is_subtype(left, item) for item in right.items)
@@ -1309,6 +1309,8 @@ def f(self) -> A: ...
13091309
if not members_right.issubset(members_left):
13101310
return False
13111311
assuming = right.type.assuming_proper if proper_subtype else right.type.assuming
1312+
if len(assuming) > MAX_PROTOCOL_DEPTH:
1313+
return True
13121314
for l, r in reversed(assuming):
13131315
if l == left and r == right:
13141316
return True
@@ -2162,21 +2164,26 @@ def unify_generic_callable(
21622164
if return_constraint_direction is None:
21632165
return_constraint_direction = mypy.constraints.SUBTYPE_OF
21642166

2165-
constraints: list[mypy.constraints.Constraint] = []
2166-
# There is some special logic for inference in callables, so better use them
2167-
# as wholes instead of picking separate arguments.
2168-
cs = mypy.constraints.infer_constraints(
2169-
type.copy_modified(ret_type=UninhabitedType()),
2170-
target.copy_modified(ret_type=UninhabitedType()),
2171-
mypy.constraints.SUBTYPE_OF,
2172-
skip_neg_op=True,
2173-
)
2174-
constraints.extend(cs)
2175-
if not ignore_return:
2176-
c = mypy.constraints.infer_constraints(
2177-
type.ret_type, target.ret_type, return_constraint_direction
2167+
# This code is hot in numerical libraries, so we use a faster inference
2168+
# algorithm (that may give slightly worse results in rare cases).
2169+
keep_unions_old = type_state.keep_unions
2170+
type_state.keep_unions = True
2171+
try:
2172+
# There is some special logic for inference in callables, so better use
2173+
# them as wholes instead of picking separate arguments.
2174+
constraints = mypy.constraints.infer_constraints(
2175+
type.copy_modified(ret_type=UninhabitedType()),
2176+
target.copy_modified(ret_type=UninhabitedType()),
2177+
mypy.constraints.SUBTYPE_OF,
2178+
skip_neg_op=True,
21782179
)
2179-
constraints.extend(c)
2180+
if not ignore_return:
2181+
c = mypy.constraints.infer_constraints(
2182+
type.ret_type, target.ret_type, return_constraint_direction
2183+
)
2184+
constraints.extend(c)
2185+
finally:
2186+
type_state.keep_unions = keep_unions_old
21802187
inferred_vars, _ = mypy.solve.solve_constraints(
21812188
type.variables, constraints, allow_polymorphic=True
21822189
)
@@ -2201,7 +2208,9 @@ def report(*args: Any) -> None:
22012208
return cast(NormalizedCallableType, applied)
22022209

22032210

2204-
def union_function_signatures(callables: list[CallableType]) -> CallableType | None:
2211+
def union_function_signatures(
2212+
callables: list[CallableType], *, simplify_unions: bool = False
2213+
) -> CallableType | None:
22052214
"""Combine a list of functions by taking the union of all the arguments and return types."""
22062215
if len(callables) == 1:
22072216
return callables[0]
@@ -2219,15 +2228,27 @@ def union_function_signatures(callables: list[CallableType]) -> CallableType | N
22192228
# confusing and ought to be re-written anyway.)
22202229
callables, variables = merge_typevars_in_callables_by_name(callables)
22212230

2222-
new_args: list[list[Type]] = [[] for _ in callables[0].arg_types]
2223-
new_kinds = list(callables[0].arg_kinds)
2231+
new_callable = callables[0].with_unpacked_kwargs().with_normalized_var_args()
2232+
new_args: list[list[Type]] = [[] for _ in new_callable.arg_types]
2233+
new_kinds = list(new_callable.arg_kinds)
2234+
new_names = list(new_callable.arg_names)
22242235
new_returns: list[Type] = []
22252236

22262237
for target in callables:
2238+
target = target.with_unpacked_kwargs().with_normalized_var_args()
22272239
# TODO: Enhance the merging logic to handle a wider variety of signatures.
2240+
# In particular, allow name-only arguments that appear in different order.
22282241
if len(new_kinds) != len(target.arg_kinds):
22292242
return None
22302243
for i, (new_kind, target_kind) in enumerate(zip(new_kinds, target.arg_kinds)):
2244+
if target.arg_names[i] != new_callable.arg_names[i]:
2245+
if target_kind.is_named():
2246+
return None
2247+
if target_kind.is_positional():
2248+
new_names[i] = None
2249+
if isinstance(target.arg_types[i], (ParamSpecType, UnpackType)):
2250+
# It is risky to put these inside a union.
2251+
return None
22312252
if new_kind == target_kind:
22322253
continue
22332254
if new_kind.is_positional() and target_kind.is_positional():
@@ -2239,12 +2260,19 @@ def union_function_signatures(callables: list[CallableType]) -> CallableType | N
22392260
new_args[i].append(arg)
22402261
new_returns.append(target.ret_type)
22412262

2242-
return callables[0].copy_modified(
2243-
arg_types=[mypy.typeops.make_simplified_union(args) for args in new_args],
2263+
if simplify_unions:
2264+
arg_types = [mypy.typeops.make_simplified_union(args) for args in new_args]
2265+
ret_type = mypy.typeops.make_simplified_union(new_returns)
2266+
else:
2267+
arg_types = [UnionType.make_union(args) for args in new_args]
2268+
ret_type = UnionType.make_union(new_returns)
2269+
2270+
return new_callable.copy_modified(
2271+
arg_types=arg_types,
22442272
arg_kinds=new_kinds,
2245-
ret_type=mypy.typeops.make_simplified_union(new_returns),
2273+
arg_names=new_names,
2274+
ret_type=ret_type,
22462275
variables=variables,
2247-
implicit=True,
22482276
)
22492277

22502278

‎mypy/test/testipc.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,20 @@ def server_multi_message_echo(q: Queue[str]) -> None:
3434
server.cleanup()
3535

3636

37+
def server_with_short_timeout(msg: str, q: Queue[str]) -> None:
38+
"""Serve a client that takes longer to speak than the connection timeout.
39+
40+
The timeout bounds accepting a connection, not the traffic after it, so the
41+
read below must wait however long the client needs.
42+
"""
43+
server = IPCServer(CONNECTION_NAME, timeout=1)
44+
q.put(server.connection_name)
45+
with server:
46+
data = server.read()
47+
server.write(data + msg)
48+
server.cleanup()
49+
50+
3751
class IPCTests(TestCase):
3852
def setUp(self) -> None:
3953
if sys.platform == "linux":
@@ -98,6 +112,27 @@ def test_multiple_messages(self) -> None:
98112
p.join()
99113
assert p.exitcode == 0
100114

115+
def test_server_timeout_does_not_apply_after_connecting(self) -> None:
116+
# A server's timeout bounds accepting a connection only. A client that is
117+
# merely slow to speak must not be mistaken for one that has hung up: that
118+
# is what killed build workers waiting on the coordinator to load the graph
119+
# (see #21484). POSIX gets this for free because accept() hands back a
120+
# blocking socket, so this mainly guards the Windows path.
121+
queue: Queue[str] = self.ctx.Queue()
122+
msg = " -- echoed"
123+
p = self.ctx.Process(target=server_with_short_timeout, args=(msg, queue), daemon=True)
124+
p.start()
125+
connection_name = queue.get()
126+
with IPCClient(connection_name, timeout=1) as client:
127+
# Stay quiet for well past the server's 1s connection timeout.
128+
time.sleep(2.5)
129+
client.write("hello")
130+
assert client.read() == "hello" + msg
131+
queue.close()
132+
queue.join_thread()
133+
p.join()
134+
assert p.exitcode == 0
135+
101136
# Run test_connect_twice a lot, in the hopes of finding issues.
102137
# This is really slow, so it is skipped, but can be enabled if
103138
# needed to debug IPC issues.

‎mypy/types.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,12 @@ class SentinelValue(NamedTuple):
222222
# A placeholder for int parameters
223223
_dummy_int: Final = -999999
224224

225+
# Maximum protocol subtyping assumptions depth. We need this to avoid infinite
226+
# recursion for protocols that are genuinely undecidable, see testDivergingProtocol.
227+
# We set a conservative cut-off, since some numerical libraries currently
228+
# use ~10 assumptions, and we want to avoid false negatives with them.
229+
MAX_PROTOCOL_DEPTH: Final = 20
230+
225231

226232
class TypeOfAny:
227233
"""

‎mypy/typestate.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,13 @@ class TypeState:
9595
# Whether to use new type inference algorithm that can infer polymorphic types.
9696
# This is temporary and will be removed soon when new algorithm is more polished.
9797
infer_polymorphic: bool
98+
# No not simplify/normalize unions during type inference in this flag is set.
99+
# Technically, union simplification gives strictly better type inference, but it
100+
# may have bad performance (since it is quadratic). So we may want to disable
101+
# it situations where performance is important.
102+
keep_unions: bool
98103

99-
# N.B: We do all of the accesses to these properties through
104+
# N.B: We do all the accesses to these properties through
100105
# TypeState, instead of making these classmethods and accessing
101106
# via the cls parameter, since mypyc can optimize accesses to
102107
# Final attributes of a directly referenced type.
@@ -113,6 +118,7 @@ def __init__(self) -> None:
113118
self.inferring = []
114119
self.infer_unions = False
115120
self.infer_polymorphic = False
121+
self.keep_unions = False
116122

117123
def is_assumed_subtype(self, left: Type, right: Type) -> bool:
118124
for l, r in reversed(self._assuming):

‎mypyc/codegen/emitclass.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1498,7 +1498,20 @@ def success() -> None:
14981498
wrapper_name = emit_instance(cl.methods["__call__"], cl.coroutine_name)
14991499
struct_name = cl.struct_name(emitter.names)
15001500
attr = emitter.attr(CPYFUNCTION_NAME)
1501-
emitter.emit_line(f"(({struct_name} *)type)->{attr} = {wrapper_name};")
1501+
if IS_FREE_THREADED:
1502+
emitter.emit_line(
1503+
f"CPy_InitAttrRefIfNull(type, (PyObject **)&(({struct_name} *)type)->{attr}, "
1504+
f"{wrapper_name});"
1505+
)
1506+
else:
1507+
# Creating the wrapper may run arbitrary code (e.g. via GC), which could
1508+
# have initialized the attribute already.
1509+
field = f"(({struct_name} *)type)->{attr}"
1510+
emitter.emit_line(f"if ({field} == NULL) {{")
1511+
emitter.emit_line(f"{field} = {wrapper_name};")
1512+
emitter.emit_line("} else {")
1513+
emitter.emit_line(f"Py_DECREF({wrapper_name});")
1514+
emitter.emit_line("}")
15021515
return success()
15031516

15041517
if not any(fn.decl.is_coroutine for fn in cl.methods.values()):

0 commit comments

Comments
 (0)