Repository navigation
Expand file tree
/
Copy path_header_generator.py
More file actions
93 lines (75 loc) · 3.74 KB
/
Copy path_header_generator.py
File metadata and controls
93 lines (75 loc) · 3.74 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Literal
from crawlee._types import HttpHeaders
from crawlee._utils.docs import docs_group
from crawlee.fingerprint_suite._browserforge_adapter import BrowserforgeHeaderGenerator
if TYPE_CHECKING:
from crawlee.fingerprint_suite._types import SupportedBrowserType
def fingerprint_browser_type_from_playwright_browser_type(
playwright_browser_type: Literal['chromium', 'firefox', 'webkit', 'chrome'],
) -> SupportedBrowserType:
if playwright_browser_type in {'chromium', 'chrome'}:
return 'chrome'
if playwright_browser_type == 'firefox':
return 'firefox'
if playwright_browser_type == 'webkit':
return 'safari'
raise ValueError(f'Unsupported browser type: {playwright_browser_type}')
@docs_group('Other')
class HeaderGenerator:
"""Generate realistic looking or browser-like HTTP headers."""
def __init__(self) -> None:
self._generator = BrowserforgeHeaderGenerator()
def _select_specific_headers(self, all_headers: dict[str, str], header_names: set[str]) -> HttpHeaders:
return HttpHeaders({key: value for key, value in all_headers.items() if key in header_names})
def get_specific_headers(
self, header_names: set[str] | None = None, browser_type: SupportedBrowserType = 'chrome'
) -> HttpHeaders:
"""Return subset of headers based on the selected `header_names`.
If no `header_names` are specified, full unfiltered headers are returned.
"""
all_headers = self._generator.generate(browser_type=browser_type)
if not header_names:
return HttpHeaders(all_headers)
return self._select_specific_headers(all_headers, header_names)
def get_common_headers(self) -> HttpHeaders:
"""Get common HTTP headers ("Accept", "Accept-Language").
Deprecated, use `get_specific_headers` instead.
We do not modify the "Accept-Encoding", "Connection" and other headers. They should be included and handled
by the HTTP client or browser.
"""
warnings.warn(
'`HeaderGenerator.get_common_headers` is deprecated and will be removed in v2.0.0. '
'Use `get_specific_headers` instead.',
DeprecationWarning,
stacklevel=2,
)
all_headers = self._generator.generate()
return self._select_specific_headers(all_headers, header_names={'Accept', 'Accept-Language'})
def get_random_user_agent_header(self) -> HttpHeaders:
"""Get a random User-Agent header."""
all_headers = self._generator.generate()
return self._select_specific_headers(all_headers, header_names={'User-Agent'})
def get_user_agent_header(
self,
*,
browser_type: SupportedBrowserType = 'chrome',
) -> HttpHeaders:
"""Get the User-Agent header based on the browser type."""
if browser_type not in {'chrome', 'firefox', 'safari', 'edge'}:
raise ValueError(f'Unsupported browser type: {browser_type}')
all_headers = self._generator.generate(browser_type=browser_type)
return self._select_specific_headers(all_headers, header_names={'User-Agent'})
def get_sec_ch_ua_headers(
self,
*,
browser_type: SupportedBrowserType = 'chrome',
) -> HttpHeaders:
"""Get the sec-ch-ua headers based on the browser type."""
if browser_type not in {'chrome', 'firefox', 'safari', 'edge'}:
raise ValueError(f'Unsupported browser type: {browser_type}')
all_headers = self._generator.generate(browser_type=browser_type)
return self._select_specific_headers(
all_headers, header_names={'sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform'}
)