Prohibit unnormalized except in stubs

gvanrossum · Mar 14, 2017 · Feb 14, 2017 · Feb 15, 2017 · Feb 15, 2017 · Feb 15, 2017
commit 06739356700b3b8c941a264543a41be3c4750868
diff --git a/mypy/nodes.py b/mypy/nodes.py
@@ -90,6 +90,15 @@ def get_column(self) -> int: pass
                             for alias, name in type_aliases.items())  # type: Dict[str, str]
 
 
+nongen_builtins = {'builtins.tuple': 'typing.Tuple',
+                   'builtins.frozenset': 'typing.FrozenSet',
+                   'builtins.enumerate': '',
+                   'collections.defaultdict': 'typing.DefaultDict',
+                   'collections.Counter': 'typing.Counter',
+                   'collections.ChainMap': 'typing.ChainMap'}
+nongen_builtins.update(reverse_type_aliases)
+
+
 # See [Note Literals and literal_hash] below
 Key = tuple
 
@@ -2143,17 +2152,20 @@ class SymbolTableNode:
     # For deserialized MODULE_REF nodes, the referenced module name;
     # for other nodes, optionally the name of the referenced object.
     cross_ref = None  # type: Optional[str]
+    # Was this node created by normaloze_type_alias?
+    normalized = False  # type: bool
 
     def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None,
                  typ: 'mypy.types.Type' = None,
                  tvar_def: 'mypy.types.TypeVarDef' = None,
-                 module_public: bool = True) -> None:
+                 module_public: bool = True, normalized: bool = False) -> None:
         self.kind = kind
         self.node = node
         self.type_override = typ
         self.mod_id = mod_id
         self.tvar_def = tvar_def
         self.module_public = module_public
+        self.normalized = normalized
 
     @property
     def fullname(self) -> str:

diff --git a/mypy/semanal.py b/mypy/semanal.py
@@ -65,7 +65,7 @@
     SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr,
     YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, AwaitExpr,
     IntExpr, FloatExpr, UnicodeExpr, EllipsisExpr, TempNode,
-    COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, LITERAL_YES,
+    COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, LITERAL_YES, nongen_builtins,
 )
 from mypy.typevars import has_no_typevars, fill_typevars
 from mypy.visitor import NodeVisitor
@@ -77,7 +77,9 @@
     TupleType, UnionType, StarType, EllipsisType, function_type, TypedDictType,
 )
 from mypy.nodes import implicit_module_attrs
-from mypy.typeanal import TypeAnalyser, TypeAnalyserPass3, analyze_type_alias
+from mypy.typeanal import (
+    TypeAnalyser, TypeAnalyserPass3, analyze_type_alias, no_subscript_builtin_alias,
+)
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
 from mypy.sametypes import is_same_type
 from mypy.options import Options
@@ -1194,7 +1196,8 @@ def visit_import_from(self, imp: ImportFrom) -> None:
                 symbol = SymbolTableNode(node.kind, node.node,
                                          self.cur_mod_id,
                                          node.type_override,
-                                         module_public=module_public)
+                                         module_public=module_public,
+                                         normalized=node.normalized)
                 self.add_symbol(imported_id, symbol, imp)
             elif module and not missing:
                 # Missing attribute.
@@ -1230,12 +1233,18 @@ def process_import_over_existing_name(self,
 
     def normalize_type_alias(self, node: SymbolTableNode,
                              ctx: Context) -> SymbolTableNode:
+        normalized = False
         if node.fullname in type_aliases:
             # Node refers to an aliased type such as typing.List; normalize.
             node = self.lookup_qualified(type_aliases[node.fullname], ctx)
+            normalized = True
         if node.fullname == 'typing.DefaultDict':
             self.add_module_symbol('collections', '__mypy_collections__', False, ctx)
             node = self.lookup_qualified('__mypy_collections__.defaultdict', ctx)
+            normalized = True
+        if normalized:
+            node = SymbolTableNode(node.kind, node.node,
+                                   node.mod_id, node.type_override, normalized=True)
         return node
 
     def correct_relative_import(self, node: Union[ImportFrom, ImportAll]) -> str:
@@ -1271,7 +1280,8 @@ def visit_import_all(self, i: ImportAll) -> None:
                             continue
                     self.add_symbol(name, SymbolTableNode(node.kind, node.node,
                                                           self.cur_mod_id,
-                                                          node.type_override), i)
+                                                          node.type_override,
+                                                          normalized=node.normalized), i)
         else:
             # Don't add any dummy symbols for 'from x import *' if 'x' is unknown.
             pass
@@ -1322,7 +1332,7 @@ def anal_type(self, t: Type, allow_tuple_literal: bool = False,
             a = TypeAnalyser(self.lookup_qualified,
                              self.lookup_fully_qualified,
                              self.fail,
-                             aliasing=aliasing)
+                             aliasing=aliasing, allow_unnormalized=self.is_stub_file)
             return t.accept(a)
         else:
             return None
@@ -1344,7 +1354,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
                 res = analyze_type_alias(s.rvalue,
                                          self.lookup_qualified,
                                          self.lookup_fully_qualified,
-                                         self.fail)
+                                         self.fail, allow_unnormalized=self.is_stub_file)
                 if res and (not isinstance(res, Instance) or res.args):
                     # TODO: What if this gets reassigned?
                     name = s.lvalues[0]
@@ -2633,7 +2643,7 @@ def visit_index_expr(self, expr: IndexExpr) -> None:
             res = analyze_type_alias(expr,
                                      self.lookup_qualified,
                                      self.lookup_fully_qualified,
-                                     self.fail)
+                                     self.fail, allow_unnormalized=self.is_stub_file)
             expr.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res),
                                           in_runtime=True)
         elif refers_to_class_or_function(expr.base):
@@ -2654,6 +2664,14 @@ def visit_index_expr(self, expr: IndexExpr) -> None:
                 types.append(typearg)
             expr.analyzed = TypeApplication(expr.base, types)
             expr.analyzed.line = expr.line
+            # list, dict, set are not directly subscriptable
+            try:
+                t = expr_to_unanalyzed_type(expr.base)
+            except TypeTranslationError:
+                return
+            n = self.lookup_qualified(t.name, expr)
+            if not n.normalized and n.fullname in nongen_builtins:
+                self.fail(no_subscript_builtin_alias(n.fullname), expr)
         else:
             expr.index.accept(self)
 

diff --git a/mypy/typeanal.py b/mypy/typeanal.py
@@ -11,7 +11,7 @@
 from mypy.nodes import (
     BOUND_TVAR, UNBOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED,
     TypeInfo, Context, SymbolTableNode, Var, Expression,
-    IndexExpr, RefExpr
+    IndexExpr, RefExpr, nongen_builtins,
 )
 from mypy.sametypes import is_same_type
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
@@ -32,7 +32,8 @@
 def analyze_type_alias(node: Expression,
                        lookup_func: Callable[[str, Context], SymbolTableNode],
                        lookup_fqn_func: Callable[[str], SymbolTableNode],
-                       fail_func: Callable[[str, Context], None]) -> Type:
+                       fail_func: Callable[[str, Context], None],
+                       allow_unnormalized: bool = False) -> Type:
     """Return type if node is valid as a type alias rvalue.
 
     Return None otherwise. 'node' must have been semantically analyzed.
@@ -67,23 +68,17 @@ def analyze_type_alias(node: Expression,
     except TypeTranslationError:
         fail_func('Invalid type alias', node)
         return None
-    analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func, aliasing=True)
+    analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func, aliasing=True,
+                            allow_unnormalized=allow_unnormalized)
     return type.accept(analyzer)
 
 
-builtin_aliases = {'tuple': 'Tuple',
-                   'list': 'List',
-                   'dict': 'Dict',
-                   'set': 'Set',
-                   'frozenset': 'FrozenSet'}
-
-def check_builtin_alian(expr)
-    for alias in builtin_aliases:
-        if refers_to_fullname(expr.base, 'builtins.' + alias):
-            self.fail('"{}" is not subscriptable, use "typing.{}" instead'
-                          .format(alias, builtin_aliases[alias]), expr)
-    if refers_to_fullname(expr.base, 'builtins.enumerate'):
-        self.fail('"enumerate" is not subscriptable', expr)
+def no_subscript_builtin_alias(name: str) -> str:
+    msg = '"{}" is not subscriptable'.format(name.split('.')[-1])
+    replacement = nongen_builtins[name]
+    if replacement:
+        msg += ', use "{}" instead'.format(replacement)
+    return msg
 
 
 class TypeAnalyser(TypeVisitor[Type]):
@@ -96,11 +91,12 @@ def __init__(self,
                  lookup_func: Callable[[str, Context], SymbolTableNode],
                  lookup_fqn_func: Callable[[str], SymbolTableNode],
                  fail_func: Callable[[str, Context], None], *,
-                 aliasing: bool = False) -> None:
+                 aliasing: bool = False, allow_unnormalized: bool = False) -> None:
         self.lookup = lookup_func
         self.lookup_fqn_func = lookup_fqn_func
         self.fail = fail_func
         self.aliasing = aliasing
+        self.allow_unnormalized = allow_unnormalized
 
     def visit_unbound_type(self, t: UnboundType) -> Type:
         if t.optional:
@@ -116,6 +112,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type:
                     self.fail('Internal error (node is None, kind={})'.format(sym.kind), t)
                 return AnyType()
             fullname = sym.node.fullname()
+            if (fullname in nongen_builtins and t.args and
+                    not sym.normalized and not self.allow_unnormalized):
+                self.fail(no_subscript_builtin_alias(fullname), t)
             if sym.kind == BOUND_TVAR:
                 if len(t.args) > 0:
                     self.fail('Type variable "{}" used with arguments'.format(