Repository navigation
Expand file tree
/
Copy pathtest_pypi_packages.py
More file actions
executable file
·92 lines (72 loc) · 2.3 KB
/
Copy pathtest_pypi_packages.py
File metadata and controls
executable file
·92 lines (72 loc) · 2.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#!/usr/bin/env python3.8
import argparse
import os
import glob
import tarfile
import zipfile
import shutil
import pathlib
import sys
from typing import Generator
sys.path.insert(0, ".")
from scripts import test_parse_directory
HERE = pathlib.Path(__file__).resolve().parent
argparser = argparse.ArgumentParser(
prog="test_pypi_packages",
description="Helper program to test parsing PyPI packages",
)
argparser.add_argument(
"-t", "--tree", action="count", help="Compare parse tree to official AST", default=0
)
def get_packages() -> Generator[str, None, None]:
all_packages = (
glob.glob("./data/pypi/*.tar.gz")
+ glob.glob("./data/pypi/*.zip")
+ glob.glob("./data/pypi/*.tgz")
)
for package in all_packages:
yield package
def extract_files(filename: str) -> None:
savedir = os.path.join("data", "pypi")
if tarfile.is_tarfile(filename):
tarfile.open(filename).extractall(savedir)
elif zipfile.is_zipfile(filename):
zipfile.ZipFile(filename).extractall(savedir)
else:
raise ValueError(f"Could not identify type of compressed file {filename}")
def find_dirname(package_name: str) -> str:
for name in os.listdir(os.path.join("data", "pypi")):
full_path = os.path.join("data", "pypi", name)
if os.path.isdir(full_path) and name in package_name:
return full_path
assert False # This is to fix mypy, should never be reached
def run_tests(dirname: str, tree: int) -> int:
return test_parse_directory.parse_directory(
dirname,
verbose=False,
excluded_files=[],
tree_arg=tree,
short=True,
mode=1 if tree else 0,
parser="pegen",
)
def main() -> None:
args = argparser.parse_args()
tree = args.tree
for package in get_packages():
print(f"Extracting files from {package}... ", end="")
try:
extract_files(package)
print("Done")
except ValueError as e:
print(e)
continue
print(f"Trying to parse all python files ... ")
dirname = find_dirname(package)
status = run_tests(dirname, tree)
if status == 0:
shutil.rmtree(dirname)
else:
print(f"Failed to parse {dirname}")
if __name__ == "__main__":
main()