-
Notifications
You must be signed in to change notification settings - Fork 237
Expand file tree
/
Copy pathstring-split.py
More file actions
executable file
·76 lines (61 loc) · 1.92 KB
/
Copy pathstring-split.py
File metadata and controls
executable file
·76 lines (61 loc) · 1.92 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright the Vortex contributors
"""
Run string-bench once per (column, encoder), drop OS cache between runs, merge
outputs
"""
import glob
import re
import subprocess
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
BINARY = "target/release_debug/string-bench"
PARTS_DIR = Path("parts")
COLUMNS = ["URL", "l_comment"]
ENCODERS = ["onpair", "fsst"]
def drop_os_caches() -> None:
try:
subprocess.run(["sync"], check=True)
subprocess.run(
["sudo", "-n", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"],
check=True,
capture_output=True,
)
except (OSError, subprocess.CalledProcessError):
pass
def run_combinations() -> None:
PARTS_DIR.mkdir(parents=True, exist_ok=True)
i = 0
for column in COLUMNS:
for encoder in ENCODERS:
drop_os_caches()
args = [
"bash",
str(SCRIPT_DIR / "bench-taskset.sh"),
BINARY,
"--columns",
f"^{re.escape(column)}$",
"--encoders",
encoder,
"-d",
"gh-json",
"-o",
str(PARTS_DIR / f"{i}.gh.json"),
]
print("+", " ".join(args), flush=True)
subprocess.run(args, check=True)
i += 1
def merge(pattern: str, out_path: str) -> None:
lines: list[str] = []
for path in sorted(glob.glob(pattern)):
with open(path, encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if line:
lines.append(line)
Path(out_path).write_text("".join(line + "\n" for line in lines), encoding="utf-8")
def main() -> None:
run_combinations()
merge(f"{PARTS_DIR}/*.gh.json", "results.json")
if __name__ == "__main__":
main()