Repository navigation
Expand file tree
/
Copy pathmodels.py
More file actions
183 lines (119 loc) · 6.78 KB
/
Copy pathmodels.py
File metadata and controls
183 lines (119 loc) · 6.78 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
from __future__ import annotations
from datetime import datetime
from typing import TYPE_CHECKING, Annotated, Any, Generic
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
from typing_extensions import TypeVar
from crawlee._types import HttpMethod, JsonSerializable
from crawlee._utils.docs import docs_group
from crawlee._utils.urls import validate_http_url
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
KvsValueType = TypeVar('KvsValueType', default=Any)
@docs_group('Storage data')
class StorageMetadata(BaseModel):
"""Represents the base model for storage metadata.
It contains common fields shared across all specific storage types.
"""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, extra='allow', from_attributes=True)
id: Annotated[str, Field(alias='id')]
"""The unique identifier of the storage."""
name: Annotated[str | None, Field(alias='name', default=None)]
"""The name of the storage."""
accessed_at: Annotated[datetime, Field(alias='accessedAt')]
"""The timestamp when the storage was last accessed."""
created_at: Annotated[datetime, Field(alias='createdAt')]
"""The timestamp when the storage was created."""
modified_at: Annotated[datetime, Field(alias='modifiedAt')]
"""The timestamp when the storage was last modified."""
@docs_group('Storage data')
class DatasetMetadata(StorageMetadata):
"""Model for a dataset metadata."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
item_count: Annotated[int, Field(alias='itemCount')]
"""The number of items in the dataset."""
@docs_group('Storage data')
class KeyValueStoreMetadata(StorageMetadata):
"""Model for a key-value store metadata."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
@docs_group('Storage data')
class RequestQueueMetadata(StorageMetadata):
"""Model for a request queue metadata."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
had_multiple_clients: Annotated[bool, Field(alias='hadMultipleClients')]
"""Indicates whether the queue has been accessed by multiple clients (consumers)."""
handled_request_count: Annotated[int, Field(alias='handledRequestCount')]
"""The number of requests that have been handled from the queue."""
pending_request_count: Annotated[int, Field(alias='pendingRequestCount')]
"""The number of requests that are still pending in the queue."""
total_request_count: Annotated[int, Field(alias='totalRequestCount')]
"""The total number of requests that have been added to the queue."""
@docs_group('Storage data')
class KeyValueStoreRecordMetadata(BaseModel):
"""Model for a key-value store record metadata."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
key: Annotated[str, Field(alias='key')]
"""The key of the record.
A unique identifier for the record in the key-value store.
"""
content_type: Annotated[str, Field(alias='contentType')]
"""The MIME type of the record.
Describe the format and type of data stored in the record, following the MIME specification.
"""
size: Annotated[int | None, Field(alias='size', default=None)] = None
"""The size of the record in bytes."""
@docs_group('Storage data')
class KeyValueStoreRecord(KeyValueStoreRecordMetadata, Generic[KvsValueType]):
"""Model for a key-value store record."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
value: Annotated[KvsValueType, Field(alias='value')]
"""The value of the record."""
@docs_group('Storage data')
class DatasetItemsListPage(BaseModel):
"""Model for a single page of dataset items returned from a collection list method."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
count: Annotated[int, Field(default=0)]
"""The number of objects returned on this page."""
offset: Annotated[int, Field(default=0)]
"""The starting position of the first object returned, as specified in the API call."""
limit: Annotated[int, Field(default=0)]
"""The maximum number of objects to return, as specified in the API call."""
total: Annotated[int, Field(default=0)]
"""The total number of objects that match the criteria of the API call."""
desc: Annotated[bool, Field(default=False)]
"""Indicates if the returned list is in descending order."""
# Workaround for Pydantic and type checkers when using Annotated with default_factory
if TYPE_CHECKING:
items: Sequence[Mapping[str, JsonSerializable]] = []
"""The list of dataset items returned on this page."""
else:
items: Annotated[list[dict], Field(default_factory=list)]
"""The list of dataset items returned on this page."""
@docs_group('Storage data')
class ProcessedRequest(BaseModel):
"""Represents a processed request."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
id: Annotated[str | None, Field(alias='requestId', default=None)] = None
"""Internal representation of the request by the storage client. Only some clients use id."""
unique_key: Annotated[str, Field(alias='uniqueKey')]
was_already_present: Annotated[bool, Field(alias='wasAlreadyPresent')]
was_already_handled: Annotated[bool, Field(alias='wasAlreadyHandled')]
@docs_group('Storage data')
class UnprocessedRequest(BaseModel):
"""Represents an unprocessed request."""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
unique_key: Annotated[str, Field(alias='uniqueKey')]
url: Annotated[str, BeforeValidator(validate_http_url), Field()]
method: Annotated[HttpMethod | None, Field()] = None
@docs_group('Storage data')
class AddRequestsResponse(BaseModel):
"""Model for a response to add requests to a queue.
Contains detailed information about the processing results when adding multiple requests
to a queue. This includes which requests were successfully processed and which ones
encountered issues during processing.
"""
model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
processed_requests: Annotated[list[ProcessedRequest], Field(alias='processedRequests')]
"""Successfully processed requests, including information about whether they were
already present in the queue and whether they had been handled previously."""
unprocessed_requests: Annotated[list[UnprocessedRequest], Field(alias='unprocessedRequests')]
"""Requests that could not be processed, typically due to validation errors or other issues."""