Skip to content

Commit cc686a2

Browse files
matthiasgoergensmatthiasgoergens
committed
Record runner CPU; add makefile_sed arms; exp12-sre config
GitHub's ubuntu-24.04 runners come with different CPUs (EPYC 7763, 9V74, 9V45, Xeon Platinum 8573C, Xeon 6973P-C were all seen), and the regex_effbot slowdown of pythongh-158286 is largest on EPYC 7763. blockbench now records the CPU model in every row, and `analyze --cpu REGEX` restricts to matching blocks. backfill_cpu.py recovers the CPU of older runs from job logs while GitHub still has them; all-cpu.jsonl for exp3, exp9b and exp11b were made with it. build_arms.py accepts "makefile_sed" to edit the generated Makefile (and fails if an expression changes nothing). exp12-sre uses it to compile Modules/_sre/sre.c with -fno-lto, to test whether keeping the regex engine's dispatch out of the link-time option avoids the regression. make_branch.sh now pushes to $PERF_CI_REMOTE without --force: in a clone where origin is python/cpython, the old `git push -f origin` would have targeted upstream. Also drop a stray .pyc and ignore __pycache__.
1 parent 4de591f commit cc686a2

10 files changed

Lines changed: 32766 additions & 4 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__pycache__/

‎ci/build_arms.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
Arm spec forms in config["arms"]:
55
"name": "<sha>" build <sha> with config["configure"]
66
"name": {"ref": "<sha>", "configure_extra": "..."} same, with extra configure flags
7+
"name": {"ref": "<sha>", "makefile_sed": ["s|a|b|", ...]}
8+
same, with sed expressions applied to the
9+
generated Makefile before make (each must
10+
change it, or the build fails)
711
"name": {"same_as": "<arm>"} same binary as <arm> (run-to-run control)
812
"name": {"same_as": "<arm>", "sitecustomize": "..."}
913
wrapper around <arm>'s binary that runs the
@@ -68,7 +72,7 @@ def sha(path):
6872
spec = {'ref': spec}
6973
if 'ref' not in spec:
7074
continue
71-
key = (spec['ref'], spec.get('configure_extra', ''))
75+
key = (spec['ref'], spec.get('configure_extra', ''), tuple(spec.get('makefile_sed', ())))
7276
bdir = f'{HOME}/b-{name}'
7377
src = f'{HOME}/src-{spec["ref"][:12]}'
7478
if not os.path.exists(src):
@@ -100,6 +104,14 @@ def sha(path):
100104
os.makedirs(bdir, exist_ok=True)
101105
try:
102106
sh(f'{src}/configure {configure} {spec.get("configure_extra", "")} > configure.log 2>&1', cwd=bdir, env=env)
107+
for expr in spec.get('makefile_sed', ()):
108+
with open(f'{bdir}/Makefile') as f:
109+
before = f.read()
110+
subprocess.run(['sed', '-i', '-e', expr, 'Makefile'], cwd=bdir, check=True)
111+
with open(f'{bdir}/Makefile') as f:
112+
if f.read() == before:
113+
sys.exit(f'{name}: makefile_sed {expr!r} did not change the Makefile')
114+
print(f'{name}: applied makefile_sed {expr!r}', flush=True)
103115
sh(f'make -j{os.cpu_count()} > make.log 2>&1', cwd=bdir, env=env)
104116
if sys.platform == 'darwin':
105117
with open(f'{arm_dir}/python', 'w') as f:

‎ci/make_branch.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env bash
22
# Create and push a throwaway perf-ci/<name> branch that runs perf-block.yml.
33
#
4-
# perf-notes/ci/make_branch.sh NAME CONFIG.json [WORKFLOW.yml]
4+
# PERF_CI_REMOTE=<fork remote> perf-notes/ci/make_branch.sh NAME CONFIG.json [WORKFLOW.yml]
55
# (WORKFLOW defaults to perf-notes/ci/perf-block.yml)
66
#
77
# CONFIG.json: {"arms": {...}, "jobs": N, "blocks_per_job": B, "scale": S, "values": V,
@@ -53,4 +53,6 @@ commit=$(git commit-tree "$tree" "${parents[@]}" -m "perf-ci: $name
5353
$(cat "$tmp/config.json")")
5454
rm -rf "$tmp"
5555
echo "commit $commit"
56-
git push -f origin "$commit:refs/heads/perf-ci/$name"
56+
# Push to an explicitly named remote (the fork), never to whatever "origin" is in
57+
# this clone, and never with --force: a perf-ci/<name> branch is created once.
58+
git push "${PERF_CI_REMOTE:?set PERF_CI_REMOTE to the fork remote}" "$commit:refs/heads/perf-ci/$name"

‎experiments/exp11b/all-cpu.jsonl‎

Lines changed: 5976 additions & 0 deletions
Large diffs are not rendered by default.

‎experiments/exp12-sre/config.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"arms": {
3+
"c19": {
4+
"ref": "6af40a682881f8442e1bdae67d34b186ef42826a",
5+
"cc": "clang-19",
6+
"configure": "--with-lto=thin"
7+
},
8+
"c19_srenolto": {
9+
"ref": "6af40a682881f8442e1bdae67d34b186ef42826a",
10+
"cc": "clang-19",
11+
"configure": "--with-lto=thin",
12+
"makefile_sed": [
13+
"s|-c $(srcdir)/Modules/_sre/sre.c -o Modules/_sre/sre.o|-fno-lto -c $(srcdir)/Modules/_sre/sre.c -o Modules/_sre/sre.o|"
14+
]
15+
},
16+
"c19patched": {
17+
"ref": "3d46d2d3f9e1b4d400dda36b26be7080ef5a0ff5",
18+
"cc": "clang-19",
19+
"configure": "--with-lto=thin"
20+
},
21+
"c19patched_srenolto": {
22+
"ref": "3d46d2d3f9e1b4d400dda36b26be7080ef5a0ff5",
23+
"cc": "clang-19",
24+
"configure": "--with-lto=thin",
25+
"makefile_sed": [
26+
"s|-c $(srcdir)/Modules/_sre/sre.c -o Modules/_sre/sre.o|-fno-lto -c $(srcdir)/Modules/_sre/sre.c -o Modules/_sre/sre.o|"
27+
]
28+
},
29+
"c19same": {
30+
"same_as": "c19"
31+
}
32+
},
33+
"jobs": 12,
34+
"blocks_per_job": 4,
35+
"rounds": 3,
36+
"values": 1,
37+
"scale": 1,
38+
"bench": "^regex_"
39+
}

‎experiments/exp3/all-cpu.jsonl‎

Lines changed: 22176 additions & 0 deletions
Large diffs are not rendered by default.

‎experiments/exp9b/all-cpu.jsonl‎

Lines changed: 4482 additions & 0 deletions
Large diffs are not rendered by default.

‎tools/__pycache__/irbench.cpython-311.pyc‎

-26.8 KB
Binary file not shown.

‎tools/backfill_cpu.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env python3
2+
"""Add the runner CPU to blockbench rows from runs made before rows recorded it.
3+
4+
backfill_cpu.py OWNER/REPO RUN_ID IN.jsonl OUT.jsonl
5+
6+
Rows are tagged "<run_id>-<matrix job>"; the CPU of each matrix job is read from
7+
the "Machine info" step of that job's log (lscpu "Model name"). Needs `gh`, and
8+
only works while GitHub still has the logs (90 days by default).
9+
"""
10+
import json
11+
import re
12+
import subprocess
13+
import sys
14+
15+
repo, run_id, src, dst = sys.argv[1:5]
16+
jobs = json.loads(subprocess.check_output(
17+
['gh', 'api', '--paginate', f'repos/{repo}/actions/runs/{run_id}/jobs?per_page=100'], text=True))['jobs']
18+
cpu_by_tag = {}
19+
for job in jobs:
20+
m = re.fullmatch(r'block \((\d+)\)', job['name'])
21+
if not m:
22+
continue
23+
p = subprocess.run(['gh', 'api', '--allow-escape-sequences',
24+
f'repos/{repo}/actions/jobs/{job["id"]}/logs'],
25+
capture_output=True, text=True)
26+
cpu = re.search(r'Model name:\s*(.+)', p.stdout)
27+
if p.returncode or not cpu:
28+
sys.exit(f'job {job["id"]} ({job["name"]}): no CPU in log '
29+
f'(gh exit {p.returncode}: {p.stderr.strip()[:200]})')
30+
cpu_by_tag[f'{run_id}-{m.group(1)}'] = cpu.group(1).strip()
31+
for tag, cpu in sorted(cpu_by_tag.items()):
32+
print(f'{tag}: {cpu}', file=sys.stderr)
33+
34+
missing = set()
35+
with open(src) as f, open(dst, 'w') as out:
36+
for line in f:
37+
if not line.strip():
38+
continue
39+
r = json.loads(line)
40+
r.setdefault('cpu', cpu_by_tag.get(r['tag'], 'unknown'))
41+
if r['tag'] not in cpu_by_tag:
42+
missing.add(r['tag'])
43+
out.write(json.dumps(r) + '\n')
44+
if missing:
45+
sys.exit(f'no CPU found for tags: {sorted(missing)}')

‎tools/blockbench.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,25 @@ def run_cell_json(python, script, extra, loops, values, tmpdir, hashseed=0):
6969
return res
7070

7171

72+
def cpu_model():
73+
"""CPU model string of this machine, recorded in every result row: GitHub
74+
runners of one label come with different CPUs, and effects can differ by CPU."""
75+
try:
76+
with open('/proc/cpuinfo') as f:
77+
for line in f:
78+
if line.startswith('model name'):
79+
return line.split(':', 1)[1].strip()
80+
except OSError:
81+
pass
82+
try:
83+
return subprocess.check_output(['sysctl', '-n', 'machdep.cpu.brand_string'], text=True).strip()
84+
except (OSError, subprocess.CalledProcessError):
85+
return 'unknown'
86+
87+
7288
def cmd_run(args):
89+
cpu = cpu_model()
90+
print(f'cpu: {cpu}', file=sys.stderr, flush=True)
7391
arms = {}
7492
for spec in args.arm:
7593
name, paths = spec.split('=', 1)
@@ -111,7 +129,7 @@ def cmd_run(args):
111129
out.write(json.dumps({
112130
'tag': args.tag, 'block': f'{args.tag}:{block}', 'bench': sub or name,
113131
'arm': arm, 'build': build, 'round': rnd, 'pos': pos, 'loops': loops_n,
114-
'hashseed': hashseed,
132+
'hashseed': hashseed, 'cpu': cpu,
115133
'values': v, 't': time.time()}) + '\n')
116134
pos += 1
117135
out.flush()
@@ -139,6 +157,16 @@ def cmd_analyze(args):
139157
for path in args.files:
140158
with open(path) as f:
141159
rows += [json.loads(l) for l in f if l.strip()]
160+
cpus = {}
161+
for r in rows:
162+
cpus[r.get('cpu', 'unrecorded')] = cpus.get(r.get('cpu', 'unrecorded'), set()) | {r['block']}
163+
print('blocks per cpu: ' + '; '.join(f'{c}: {len(b)}' for c, b in sorted(cpus.items())))
164+
if args.cpu:
165+
pat = re.compile(args.cpu)
166+
rows = [r for r in rows if pat.search(r.get('cpu', 'unrecorded'))]
167+
print(f'--cpu {args.cpu!r}: {len({r["block"] for r in rows})} blocks kept')
168+
if not rows:
169+
sys.exit('no rows match --cpu')
142170
# response per cell: median of values; average log over builds of an arm in a block
143171
cell = {}
144172
for r in rows:
@@ -222,6 +250,7 @@ def main():
222250
a = sp.add_parser('analyze')
223251
a.add_argument('files', nargs='+')
224252
a.add_argument('--base', default='base')
253+
a.add_argument('--cpu', help='only blocks whose recorded CPU model matches this regex')
225254
a.add_argument('-q', '--quiet', action='store_true')
226255
args = ap.parse_args()
227256
{'run': cmd_run, 'analyze': cmd_analyze}[args.cmd](args)

0 commit comments

Comments
 (0)