Skip to content

Commit c4ff2cd

Browse files
committed
init
1 parent f197172 commit c4ff2cd

2 files changed

Lines changed: 138 additions & 0 deletions

File tree

‎selenium/ReadMe.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
3+
#### 环境准备
4+
```shell
5+
python3 -m venv .venv
6+
source .venv/bin/activate
7+
pip install -U pip selenium
8+
```
9+
10+
11+
#### 运行
12+
```shell
13+
python main.py --url "https://en.macromicro.me/series/22748/cnn-fear-and-greed" --out "downloads/cnn-fg.mhtml"
14+
15+
# 打开生成文件
16+
open -a "Google Chrome" downloads/xxx.mhtml
17+
```

‎selenium/main.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import argparse
2+
import os
3+
import sys
4+
import time
5+
from datetime import datetime
6+
7+
from selenium import webdriver
8+
from selenium.webdriver.chrome.options import Options
9+
from selenium.webdriver.support.ui import WebDriverWait
10+
11+
12+
def wait_for_ready(driver, timeout: int = 20):
13+
WebDriverWait(driver, timeout).until(
14+
lambda d: d.execute_script("return document.readyState") == "complete"
15+
)
16+
17+
18+
def auto_scroll(driver, pause: float = 0.3, max_steps: int = 50):
19+
last_height = driver.execute_script("return document.body.scrollHeight")
20+
steps = 0
21+
while steps < max_steps:
22+
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
23+
time.sleep(pause)
24+
new_height = driver.execute_script("return document.body.scrollHeight")
25+
if new_height == last_height:
26+
break
27+
last_height = new_height
28+
steps += 1
29+
driver.execute_script("window.scrollTo(0, 0)")
30+
31+
32+
def save_as_mhtml(driver, out_path: str):
33+
# 使用 Chrome DevTools Protocol 保存为单文件 MHTML
34+
driver.execute_cdp_cmd("Page.enable", {})
35+
result = driver.execute_cdp_cmd("Page.captureSnapshot", {"format": "mhtml"})
36+
data = result.get("data") if isinstance(result, dict) else result
37+
if not data:
38+
raise RuntimeError("Failed to capture MHTML snapshot")
39+
40+
os.makedirs(os.path.dirname(out_path), exist_ok=True)
41+
with open(out_path, "w", encoding="utf-8") as f:
42+
f.write(data)
43+
44+
45+
def build_driver(headless: bool = True) -> webdriver.Chrome:
46+
options = Options()
47+
if headless:
48+
# 新版 headless,渲染效果更接近非无头
49+
options.add_argument("--headless=new")
50+
options.add_argument("--disable-gpu")
51+
options.add_argument("--window-size=1920,1080")
52+
options.add_argument("--no-sandbox")
53+
options.add_argument("--disable-dev-shm-usage")
54+
# 让 Selenium 4+ 自动用 Selenium Manager 管理 ChromeDriver(无需 webdriver_manager)
55+
return webdriver.Chrome(options=options)
56+
57+
58+
def default_out_path(url: str) -> str:
59+
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
60+
safe_title = url.rstrip("/").split("/")[-1] or "page"
61+
return os.path.join(
62+
os.path.dirname(__file__),
63+
"downloads",
64+
f"{safe_title}-{ts}.mhtml",
65+
)
66+
67+
68+
def main(argv=None):
69+
parser = argparse.ArgumentParser(
70+
description="Download a web page with Selenium and save as a complete MHTML file."
71+
)
72+
parser.add_argument(
73+
"--url",
74+
default="https://en.macromicro.me/series/22748/cnn-fear-and-greed",
75+
help="Target page URL",
76+
)
77+
parser.add_argument(
78+
"--out",
79+
default=None,
80+
help="Output .mhtml file path (default: selenium/downloads/...)",
81+
)
82+
parser.add_argument(
83+
"--delay",
84+
type=float,
85+
default=0.0,
86+
help="Extra seconds to wait before capture (for manual actions like cookie consent)",
87+
)
88+
parser.add_argument(
89+
"--no-headless",
90+
action="store_true",
91+
help="Run browser in non-headless mode",
92+
)
93+
94+
args = parser.parse_args(argv)
95+
out_path = args.out or default_out_path(args.url)
96+
97+
driver = None
98+
try:
99+
driver = build_driver(headless=not args.no_headless)
100+
driver.get(args.url)
101+
102+
# 页面就绪
103+
wait_for_ready(driver, timeout=25)
104+
105+
# 可选:滚动触发懒加载,以便资源内嵌到 MHTML
106+
time.sleep(2)
107+
auto_scroll(driver, pause=0.4, max_steps=60)
108+
109+
# 再等待一小会儿,给异步资源加载时间;可叠加 --delay
110+
time.sleep(2 + max(0.0, args.delay))
111+
112+
save_as_mhtml(driver, out_path)
113+
print(out_path)
114+
finally:
115+
if driver is not None:
116+
driver.quit()
117+
118+
119+
if __name__ == "__main__":
120+
sys.exit(main())
121+

0 commit comments

Comments
 (0)