Set decl max size per file.

ericsnowcurrently · Dec 1, 2021 · Nov 30, 2021 · Nov 30, 2021 · Nov 30, 2021 · Nov 30, 2021
commit 4c78fb7b8a319398a75c640bdf80538ae2331ccd
@@ -1,27 +1,37 @@
+from c_common.fsutil import match_glob as _match_glob
 from .parser import parse as _parse
 from .preprocessor import get_preprocessor as _get_preprocessor
 
 
 def parse_file(filename, *,
                match_kind=None,
                get_file_preprocessor=None,
+               file_maxsizes=None,
                ):
     if get_file_preprocessor is None:
         get_file_preprocessor = _get_preprocessor()
-    yield from _parse_file(filename, match_kind, get_file_preprocessor)
+    yield from _parse_file(
+            filename, match_kind, get_file_preprocessor, file_maxsizes)
 
 
 def parse_files(filenames, *,
                 match_kind=None,
                 get_file_preprocessor=None,
+                file_maxsizes=None,
                 ):
     if get_file_preprocessor is None:
         get_file_preprocessor = _get_preprocessor()
     for filename in filenames:
-        yield from _parse_file(filename, match_kind, get_file_preprocessor)
+        yield from _parse_file(
+                filename, match_kind, get_file_preprocessor, file_maxsizes)
 
 
-def _parse_file(filename, match_kind, get_file_preprocessor):
+def _parse_file(filename, match_kind, get_file_preprocessor, maxsizes):
+    srckwargs = {}
+    maxsize = _resolve_max_size(filename, maxsizes)
+    if maxsize:
+        srckwargs['maxtext'], srckwargs['maxlines'] = maxsize
+
     # Preprocess the file.
     preprocess = get_file_preprocessor(filename)
     preprocessed = preprocess()
@@ -30,14 +40,30 @@ def _parse_file(filename, match_kind, get_file_preprocessor):
 
     # Parse the lines.
     srclines = ((l.file, l.data) for l in preprocessed if l.kind == 'source')
-    for item in _parse(srclines):
+    for item in _parse(srclines, **srckwargs):
         if match_kind is not None and not match_kind(item.kind):
             continue
         if not item.filename:
             raise NotImplementedError(repr(item))
         yield item
 
 
+def _resolve_max_size(filename, maxsizes):
+    for pattern, maxsize in (maxsizes.items() if maxsizes else ()):
+        if _match_glob(filename, pattern):
+            break
+    else:
+        return None
+    if not maxsize:
+        return None, None
+    maxtext, maxlines = maxsize
+    if maxtext is not None:
+        maxtext = int(maxtext)
+    if maxlines is not None:
+        maxlines = int(maxlines)
+    return maxtext, maxlines
+
+
 def parse_signature(text):
     raise NotImplementedError
 

@@ -120,12 +120,12 @@
 from ._info import SourceInfo
 
 
-def parse(srclines):
+def parse(srclines, **srckwargs):
     if isinstance(srclines, str):  # a filename
         raise NotImplementedError
 
     anon_name = anonymous_names()
-    for result in _parse(srclines, anon_name):
+    for result in _parse(srclines, anon_name, **srckwargs):
         yield ParsedItem.from_raw(result)
 
 
@@ -152,16 +152,18 @@ def anon_name(prefix='anon-'):
 _logger = logging.getLogger(__name__)
 
 
-def _parse(srclines, anon_name):
+def _parse(srclines, anon_name, **srckwargs):
     from ._global import parse_globals
 
-    source = _iter_source(srclines)
-    #source = _iter_source(srclines, showtext=True)
+    source = _iter_source(srclines, **srckwargs)
     for result in parse_globals(source, anon_name):
         # XXX Handle blocks here instead of in parse_globals().
         yield result
 
 
+# We use defaults that cover most files.  Files with bigger declarations
+# are covered elsewhere (MAX_SIZES in cpython/_parser.py).
+
 def _iter_source(lines, *, maxtext=20_000, maxlines=700, showtext=False):
     maxtext = maxtext if maxtext and maxtext > 0 else None
     maxlines = maxlines if maxlines and maxlines > 0 else None

@@ -111,6 +111,7 @@ def cmd_parse(filenames=None, **kwargs):
     c_parser.cmd_parse(
         filenames,
         relroot=REPO_ROOT,
+        file_maxsizes=_parser.MAX_SIZES,
         **kwargs
     )
 
@@ -127,6 +128,7 @@ def cmd_check(filenames=None, **kwargs):
         relroot=REPO_ROOT,
         _analyze=_analyzer.analyze,
         _CHECKS=CHECKS,
+        file_maxsizes=_parser.MAX_SIZES,
         **kwargs
     )
 
@@ -141,6 +143,7 @@ def cmd_analyze(filenames=None, **kwargs):
         relroot=REPO_ROOT,
         _analyze=_analyzer.analyze,
         formats=formats,
+        file_maxsizes=_parser.MAX_SIZES,
         **kwargs
     )
 

@@ -14,6 +14,10 @@
 GLOB_ALL = '**/*'
 
 
+def _abs(relfile):
+    return os.path.join(REPO_ROOT, relfile)
+
+
 def clean_lines(text):
     """Clear out comments, blank lines, and leading/trailing whitespace."""
     lines = (line.strip() for line in text.splitlines())
@@ -22,7 +26,7 @@ def clean_lines(text):
              if line and not line.startswith('#'))
     glob_all = f'{GLOB_ALL} '
     lines = (re.sub(r'^[*] ', glob_all, line) for line in lines)
-    lines = (os.path.join(REPO_ROOT, line) for line in lines)
+    lines = (_abs(line) for line in lines)
     return list(lines)
 
 
@@ -244,6 +248,7 @@ def clean_lines(text):
 Modules/sre_lib.h	LOCAL(type)	static inline type
 Modules/sre_lib.h	SRE(F)	sre_ucs2_##F
 Objects/stringlib/codecs.h	STRINGLIB_IS_UNICODE	1
+Include/internal/pycore_bitutils.h	_Py__has_builtin(B)	0
 
 # @end=tsv@
 ''')[1:]
@@ -264,6 +269,9 @@ def clean_lines(text):
     './Include/cpython/',
 ]
 
+MAX_SIZES = {
+}
+
 
 def get_preprocessor(*,
                      file_macros=None,
@@ -298,6 +306,7 @@ def parse_file(filename, *,
         filename,
         match_kind=match_kind,
         get_file_preprocessor=get_file_preprocessor,
+        file_maxsizes=MAX_SIZES,
     )
 
 
@@ -317,5 +326,6 @@ def parse_files(filenames=None, *,
         filenames,
         match_kind=match_kind,
         get_file_preprocessor=get_file_preprocessor,
+        file_maxsizes=MAX_SIZES,
         **file_kwargs
     )