|
| 1 | +"""Which updates to a built set are worth taking, and which are not yet. |
| 2 | +
|
| 3 | + python check_updates.py # newest 3.14 slim lockfile |
| 4 | + python check_updates.py --lock <pylock.toml> --min-days 21 |
| 5 | +
|
| 6 | +Answers the two questions that otherwise mean reading `pip list -o` by eye: |
| 7 | +
|
| 8 | + security the installed version has an advisory. Always shown, whatever its |
| 9 | + age, with the version that fixes it. |
| 10 | + ready something newer exists, it has stood at least --min-days without |
| 11 | + being replaced, it is not yanked, and it still has a wheel for |
| 12 | + this target. Standing unpatched is the free half of "the ecosystem |
| 13 | + moved to it"; the download half needs an API key and is not here. |
| 14 | + too new newer exists but has not aged yet -- shown with the wait left, so |
| 15 | + a version that is nearly ripe is not silently dropped. |
| 16 | +
|
| 17 | +Everything comes from PyPI's public JSON: two calls per package, no key. The |
| 18 | +installed set comes from a lockfile, so this describes a distribution that was |
| 19 | +actually built rather than whatever is installed here. |
| 20 | +""" |
| 21 | +from __future__ import annotations |
| 22 | + |
| 23 | +import argparse |
| 24 | +import json |
| 25 | +import re |
| 26 | +import sys |
| 27 | +import time |
| 28 | +import tomllib |
| 29 | +import urllib.error |
| 30 | +import urllib.request |
| 31 | +from concurrent.futures import ThreadPoolExecutor |
| 32 | +from datetime import datetime, timezone |
| 33 | +from pathlib import Path |
| 34 | + |
| 35 | +PYPI = "https://pypi.org/pypi" |
| 36 | +TIMEOUT = 30 |
| 37 | +QUOTES = "\"'" |
| 38 | +PRERELEASE = re.compile(r"(a|b|rc|dev)\d", re.I) |
| 39 | + |
| 40 | + |
| 41 | +def version_key(text: str): |
| 42 | + """Sortable version key, tolerating anything unparseable.""" |
| 43 | + try: |
| 44 | + from packaging.version import InvalidVersion, Version |
| 45 | + try: |
| 46 | + return (1, Version(text)) |
| 47 | + except InvalidVersion: |
| 48 | + pass |
| 49 | + except ImportError: |
| 50 | + pass |
| 51 | + return (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[._-]", text))) |
| 52 | + |
| 53 | + |
| 54 | +def is_prerelease(text: str) -> bool: |
| 55 | + try: |
| 56 | + from packaging.version import InvalidVersion, Version |
| 57 | + try: |
| 58 | + return Version(text).is_prerelease |
| 59 | + except InvalidVersion: |
| 60 | + pass |
| 61 | + except ImportError: |
| 62 | + pass |
| 63 | + return bool(PRERELEASE.search(text)) |
| 64 | + |
| 65 | + |
| 66 | +def fetch(url: str, attempts: int = 3): |
| 67 | + """PyPI JSON, retried: one dropped connection must not read as 'no such package'.""" |
| 68 | + for attempt in range(attempts): |
| 69 | + try: |
| 70 | + with urllib.request.urlopen(url, timeout=TIMEOUT) as response: |
| 71 | + return json.load(response) |
| 72 | + except urllib.error.HTTPError as exc: |
| 73 | + if exc.code == 404: |
| 74 | + return None |
| 75 | + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError): |
| 76 | + pass |
| 77 | + if attempt + 1 < attempts: |
| 78 | + time.sleep(1 + attempt) |
| 79 | + return None |
| 80 | + |
| 81 | + |
| 82 | +def released(files: list[dict]) -> datetime | None: |
| 83 | + stamps = [f["upload_time_iso_8601"] for f in files if f.get("upload_time_iso_8601")] |
| 84 | + if not stamps: |
| 85 | + return None |
| 86 | + return datetime.fromisoformat(min(stamps).replace("Z", "+00:00")) |
| 87 | + |
| 88 | + |
| 89 | +def wheel_note(files: list[dict], target: str) -> str: |
| 90 | + """How this release covers the target interpreter, e.g. cp314 or cp314t.""" |
| 91 | + names = [f["filename"] for f in files if f["filename"].endswith(".whl")] |
| 92 | + if not names: |
| 93 | + return "sdist only" |
| 94 | + if any(f"-{target}-" in n and "win_amd64" in n for n in names): |
| 95 | + return "wheel" |
| 96 | + if any("abi3" in n and "win_amd64" in n for n in names): |
| 97 | + return "abi3" if target.endswith("t") else "wheel" |
| 98 | + if any("py3-none-any" in n or "py2.py3-none-any" in n for n in names): |
| 99 | + return "pure" |
| 100 | + return "no win wheel" |
| 101 | + |
| 102 | + |
| 103 | +def inspect(name: str, installed: str, target: str, now: datetime) -> dict: |
| 104 | + row = {"name": name, "installed": installed, "state": "unknown", |
| 105 | + "candidate": "", "days": 0, "since": 0, "wheels": "", "note": "", |
| 106 | + "requires": []} |
| 107 | + |
| 108 | + meta = fetch(f"{PYPI}/{name}/json") |
| 109 | + vuln = fetch(f"{PYPI}/{name}/{installed}/json") |
| 110 | + if meta is None: |
| 111 | + row["note"] = "not on PyPI (or fetch failed)" |
| 112 | + return row |
| 113 | + |
| 114 | + row["requires"] = ((vuln or {}).get("info") or {}).get("requires_dist") or [] |
| 115 | + advisories = (vuln or {}).get("vulnerabilities") or [] |
| 116 | + releases = meta.get("releases", {}) |
| 117 | + usable = {v: f for v, f in releases.items() |
| 118 | + if f and not is_prerelease(v) and not all(x.get("yanked") for x in f)} |
| 119 | + newer = {v: f for v, f in usable.items() if version_key(v) > version_key(installed)} |
| 120 | + |
| 121 | + if advisories: |
| 122 | + fixes = sorted({v for a in advisories for v in (a.get("fixed_in") or [])}, |
| 123 | + key=version_key) |
| 124 | + row["state"] = "security" |
| 125 | + row["candidate"] = next((v for v in fixes if version_key(v) > version_key(installed)), |
| 126 | + max(newer, key=version_key) if newer else "") |
| 127 | + if not row["candidate"]: |
| 128 | + row["note"] = "NO FIXED VERSION PUBLISHED -- " |
| 129 | + row["note"] += ", ".join(sorted({a.get("id", "?") for a in advisories}))[:70] |
| 130 | + elif not newer: |
| 131 | + row["state"] = "current" |
| 132 | + return row |
| 133 | + else: |
| 134 | + row["candidate"] = max(newer, key=version_key) |
| 135 | + row["state"] = "ready" |
| 136 | + |
| 137 | + if row["candidate"] and row["candidate"] in releases: |
| 138 | + when = released(releases[row["candidate"]]) |
| 139 | + if when: |
| 140 | + row["days"] = (now - when).days |
| 141 | + row["wheels"] = wheel_note(releases[row["candidate"]], target) |
| 142 | + row["since"] = len(newer) |
| 143 | + return row |
| 144 | + |
| 145 | + |
| 146 | +def caps_from(rows: list[dict]) -> dict[str, list[tuple]]: |
| 147 | + """Who caps whom, extras included. |
| 148 | +
|
| 149 | + A cap hidden behind an extra is the kind that is hardest to see by hand -- |
| 150 | + it is absent from constraints.txt, from `pip list -o`, and from a reverse |
| 151 | + dependency tree that evaluates markers with no extra set. It is exactly the |
| 152 | + kind that quietly holds a package down, so record it and say who. |
| 153 | + """ |
| 154 | + try: |
| 155 | + from packaging.requirements import InvalidRequirement, Requirement |
| 156 | + except ImportError: |
| 157 | + return {} |
| 158 | + caps: dict[str, list[tuple]] = {} |
| 159 | + for row in rows: |
| 160 | + for text in row["requires"]: |
| 161 | + try: |
| 162 | + requirement = Requirement(text) |
| 163 | + except InvalidRequirement: |
| 164 | + continue |
| 165 | + if not requirement.specifier: |
| 166 | + continue |
| 167 | + marker = str(requirement.marker) if requirement.marker else "" |
| 168 | + extra = "" |
| 169 | + if "extra" in marker: |
| 170 | + # markers stringify as: extra == "autopep8" |
| 171 | + extra = marker.split("==")[-1].strip().strip(QUOTES) if "==" in marker else "?" |
| 172 | + caps.setdefault(normalize(requirement.name), []).append( |
| 173 | + (row["name"], row["installed"], requirement.specifier, extra, |
| 174 | + set(requirement.extras))) |
| 175 | + return caps |
| 176 | + |
| 177 | + |
| 178 | +def normalize(name: str) -> str: |
| 179 | + return re.sub(r"[-_.]+", "-", name).lower() |
| 180 | + |
| 181 | + |
| 182 | +def main(argv: list[str]) -> int: |
| 183 | + parser = argparse.ArgumentParser(description=__doc__, |
| 184 | + formatter_class=argparse.RawDescriptionHelpFormatter) |
| 185 | + parser.add_argument("--lock", type=Path, help="pylock.toml describing the built set") |
| 186 | + parser.add_argument("--target", default="cp314", help="interpreter tag to check wheels for") |
| 187 | + parser.add_argument("--min-days", type=int, default=21, |
| 188 | + help="how long a release must have stood to count as ready") |
| 189 | + parser.add_argument("--jobs", type=int, default=12) |
| 190 | + parser.add_argument("--out", type=Path, default=Path(".")) |
| 191 | + args = parser.parse_args(argv) |
| 192 | + |
| 193 | + lock = args.lock |
| 194 | + if lock is None: |
| 195 | + candidates = sorted(Path("winpython/portable").glob("cycle_*/pylock.64-*slim*.toml")) |
| 196 | + if not candidates: |
| 197 | + print("no slim lockfile found; pass --lock", file=sys.stderr) |
| 198 | + return 2 |
| 199 | + lock = candidates[-1] |
| 200 | + if not lock.is_file(): |
| 201 | + print(f"no such lockfile: {lock}", file=sys.stderr) |
| 202 | + return 2 |
| 203 | + |
| 204 | + data = tomllib.loads(lock.read_text(encoding="utf-8")) |
| 205 | + installed = {p["name"]: p["version"] for p in data.get("packages", [])} |
| 206 | + now = datetime.now(timezone.utc) |
| 207 | + print(f"{lock.name}: {len(installed)} packages, target {args.target}, " |
| 208 | + f"ripe at {args.min_days} days\nquerying PyPI...", flush=True) |
| 209 | + |
| 210 | + with ThreadPoolExecutor(max_workers=args.jobs) as pool: |
| 211 | + rows = list(pool.map(lambda kv: inspect(*kv, args.target, now), installed.items())) |
| 212 | + |
| 213 | + caps = caps_from(rows) |
| 214 | + for row in rows: |
| 215 | + if row["state"] == "ready" and row["days"] < args.min_days: |
| 216 | + row["state"] = "too new" |
| 217 | + if row["state"] in ("ready", "security") and row["candidate"]: |
| 218 | + blockers = [(who, ver, spec, extra) |
| 219 | + for who, ver, spec, extra, _ in caps.get(normalize(row["name"]), []) |
| 220 | + if not spec.contains(row["candidate"], prereleases=True)] |
| 221 | + if blockers: |
| 222 | + who, ver, spec, extra = blockers[0] |
| 223 | + row["state"] = "blocked" |
| 224 | + via = f"[{extra}]" if extra else "" |
| 225 | + # one hop further: the capping package is rarely the one to argue |
| 226 | + # with -- name whoever asked for it, since that is what must change. |
| 227 | + # When the cap is gated on an extra, only a dependant that asked |
| 228 | + # for that extra actually triggers it. |
| 229 | + asked = [w for w, _, _, _, extras in caps.get(normalize(who), []) |
| 230 | + if w.lower() != row["name"].lower() |
| 231 | + and (not extra or extra in extras)] |
| 232 | + if asked: |
| 233 | + origin = f", pulled in by {asked[0]}" |
| 234 | + elif extra: |
| 235 | + # nothing in the set asks for that extra, so the cap is |
| 236 | + # declared but probably inert -- worth checking, not obeying |
| 237 | + origin = f", but nothing requests [{extra}] -- may not bind" |
| 238 | + else: |
| 239 | + origin = "" |
| 240 | + row["note"] = (f"capped by {who} {ver}{via} requiring {row['name']}{spec}{origin}" |
| 241 | + + (f" -- {row['note']}" if row["note"] else "")) |
| 242 | + |
| 243 | + order = {"security": 0, "ready": 1, "blocked": 2, "too new": 3, "unknown": 4, "current": 5} |
| 244 | + rows.sort(key=lambda r: (order[r["state"]], -r["days"], r["name"])) |
| 245 | + groups = {s: [r for r in rows if r["state"] == s] for s in order} |
| 246 | + |
| 247 | + lines = [f"# update review for {lock.name}, generated {now:%Y-%m-%d}", |
| 248 | + f"# {len(installed)} packages; ready = stood {args.min_days}+ days unreplaced", |
| 249 | + "#", |
| 250 | + "# Data lines are constraint bumps, ready to paste; everything else is a", |
| 251 | + "# comment. 'since' counts releases published after the installed one --", |
| 252 | + "# a high count means the project moves fast, not that this one is risky.", |
| 253 | + ""] |
| 254 | + for state, title in (("security", "SECURITY -- advisory against the installed version"), |
| 255 | + ("ready", "READY -- aged, unreplaced, still has a wheel"), |
| 256 | + ("blocked", "BLOCKED -- something in the set caps it"), |
| 257 | + ("too new", "TOO NEW -- revisit when aged"), |
| 258 | + ("unknown", "COULD NOT CHECK")): |
| 259 | + entries = groups[state] |
| 260 | + lines += ["", f"# --- {title} ({len(entries)})"] |
| 261 | + if not entries: |
| 262 | + lines += ["# none"] |
| 263 | + continue |
| 264 | + for r in entries: |
| 265 | + detail = f"{r['installed']} -> {r['candidate']}, {r['days']}d, {r['wheels']}" |
| 266 | + if r["since"] > 1: |
| 267 | + detail += f", {r['since']} releases since" |
| 268 | + if r["note"]: |
| 269 | + detail += f", {r['note']}" |
| 270 | + prefix = "" if state in ("security", "ready") else "# " |
| 271 | + lines += [f"{prefix}{r['name']}>={r['candidate']} # {detail}" |
| 272 | + if r["candidate"] else f"# {r['name']} # {detail}"] |
| 273 | + |
| 274 | + out = args.out / f"updates.{lock.stem.replace('pylock.', '')}.txt" |
| 275 | + out.write_text("\n".join(lines) + "\n", encoding="utf-8", newline="\n") |
| 276 | + |
| 277 | + for state in order: |
| 278 | + print(f" {state:<9}: {len(groups[state]):>4}") |
| 279 | + for r in groups["security"]: |
| 280 | + print(f" SECURITY {r['name']} {r['installed']} -> {r['candidate']} {r['note']}") |
| 281 | + print(f"\nwrote {out}") |
| 282 | + return 0 |
| 283 | + |
| 284 | + |
| 285 | +if __name__ == "__main__": |
| 286 | + raise SystemExit(main(sys.argv[1:])) |
0 commit comments