Skip to content

Commit 6b001b8

Browse files
generalljoein
generall
and
joein
committed
V1.9.x (#608)
* update models * up backward compatibility version * refactoring: address review comment --------- Co-authored-by: George Panchuk <george.panchuk@qdrant.tech>
1 parent e0820e0 commit 6b001b8

7 files changed

Lines changed: 302 additions & 168 deletions

File tree

‎.github/workflows/integration-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
- name: Backward compatibility integration tests
5454
run: |
5555
export RUNNER_OS=${{ runner.os }}
56-
QDRANT_VERSION='v1.7.4' ./tests/integration-tests.sh
56+
QDRANT_VERSION='v1.8.4' ./tests/integration-tests.sh
5757
shell: bash
5858
- name: Run fastembed tests without fastembed
5959
run: |

‎qdrant_client/conversions/conversion.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ def convert_collection_info(cls, model: grpc.CollectionInfo) -> rest.CollectionI
214214
payload_schema=cls.convert_payload_schema(model.payload_schema),
215215
segments_count=model.segments_count,
216216
status=cls.convert_collection_status(model.status),
217-
vectors_count=model.vectors_count,
217+
vectors_count=model.vectors_count if model.HasField("vectors_count") else None,
218218
points_count=model.points_count,
219219
indexed_vectors_count=model.indexed_vectors_count or 0,
220220
)
@@ -380,8 +380,10 @@ def convert_collection_status(cls, model: grpc.CollectionStatus) -> rest.Collect
380380
return rest.CollectionStatus.YELLOW
381381
elif model == grpc.CollectionStatus.Red:
382382
return rest.CollectionStatus.RED
383-
else:
384-
raise ValueError(f"invalid CollectionStatus model: {model}") # pragma: no cover
383+
elif model == grpc.CollectionStatus.Grey:
384+
return rest.CollectionStatus.GREY
385+
386+
raise ValueError(f"invalid CollectionStatus model: {model}") # pragma: no cover
385387

386388
@classmethod
387389
def convert_update_result(cls, model: grpc.UpdateResult) -> rest.UpdateResult:
@@ -756,6 +758,15 @@ def convert_snapshot_description(
756758
size=model.size,
757759
)
758760

761+
@classmethod
762+
def convert_datatype(cls, model: grpc.Datatype) -> rest.Datatype:
763+
if model == grpc.Datatype.Float32:
764+
return rest.Datatype.FLOAT32
765+
elif model == grpc.Datatype.Uint8:
766+
return rest.Datatype.UINT8
767+
else:
768+
raise ValueError(f"invalid Datatype model: {model}")
769+
759770
@classmethod
760771
def convert_vector_params(cls, model: grpc.VectorParams) -> rest.VectorParams:
761772
return rest.VectorParams(
@@ -772,6 +783,7 @@ def convert_vector_params(cls, model: grpc.VectorParams) -> rest.VectorParams:
772783
else None
773784
),
774785
on_disk=model.on_disk if model.HasField("on_disk") else None,
786+
datatype=cls.convert_datatype(model.datatype) if model.HasField("datatype") else None,
775787
)
776788

777789
@classmethod
@@ -1517,7 +1529,7 @@ def convert_collection_info(cls, model: rest.CollectionInfo) -> grpc.CollectionI
15171529
),
15181530
segments_count=model.segments_count,
15191531
status=cls.convert_collection_status(model.status),
1520-
vectors_count=model.vectors_count,
1532+
vectors_count=model.vectors_count if model.vectors_count is not None else None,
15211533
points_count=model.points_count,
15221534
)
15231535

@@ -1529,6 +1541,8 @@ def convert_collection_status(cls, model: rest.CollectionStatus) -> grpc.Collect
15291541
return grpc.CollectionStatus.Yellow
15301542
if model == rest.CollectionStatus.GREEN:
15311543
return grpc.CollectionStatus.Green
1544+
if model == rest.CollectionStatus.GREY:
1545+
return grpc.CollectionStatus.Grey
15321546

15331547
raise ValueError(f"invalid CollectionStatus model: {model}") # pragma: no cover
15341548

@@ -2118,6 +2132,15 @@ def convert_snapshot_description(
21182132
size=model.size,
21192133
)
21202134

2135+
@classmethod
2136+
def convert_datatype(cls, model: rest.Datatype) -> grpc.Datatype:
2137+
if model == rest.Datatype.FLOAT32:
2138+
return grpc.Datatype.Float32
2139+
if model == rest.Datatype.UINT8:
2140+
return grpc.Datatype.Uint8
2141+
2142+
raise ValueError(f"invalid Datatype model: {model}") # pragma: no cover
2143+
21212144
@classmethod
21222145
def convert_vector_params(cls, model: rest.VectorParams) -> grpc.VectorParams:
21232146
return grpc.VectorParams(
@@ -2134,6 +2157,7 @@ def convert_vector_params(cls, model: rest.VectorParams) -> grpc.VectorParams:
21342157
else None
21352158
),
21362159
on_disk=model.on_disk,
2160+
datatype=cls.convert_datatype(model.datatype) if model.datatype is not None else None,
21372161
)
21382162

21392163
@classmethod

‎qdrant_client/grpc/collections_pb2.py‎

Lines changed: 174 additions & 156 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎qdrant_client/http/models/models.py‎

Lines changed: 61 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,14 @@
1010
VectorsConfigDiff = Dict[str, "VectorParamsDiff"]
1111

1212

13+
class AbortShardTransfer(BaseModel, extra="forbid"):
14+
shard_id: int = Field(..., description="")
15+
to_peer_id: int = Field(..., description="")
16+
from_peer_id: int = Field(..., description="")
17+
18+
1319
class AbortTransferOperation(BaseModel, extra="forbid"):
14-
abort_transfer: "MoveShard" = Field(..., description="")
20+
abort_transfer: "AbortShardTransfer" = Field(..., description="")
1521

1622

1723
class AliasDescription(BaseModel):
@@ -24,6 +30,7 @@ class AppBuildTelemetry(BaseModel):
2430
version: str = Field(..., description="")
2531
features: Optional["AppFeaturesTelemetry"] = Field(default=None, description="")
2632
system: Optional["RunningEnvironmentTelemetry"] = Field(default=None, description="")
33+
jwt_rbac: Optional[bool] = Field(default=None, description="")
2734
startup: Union[datetime, date] = Field(..., description="")
2835

2936

@@ -153,7 +160,7 @@ class CollectionInfo(BaseModel):
153160
)
154161
vectors_count: Optional[int] = Field(
155162
default=None,
156-
description="Approximate number of vectors in collection. All vectors in collection are available for querying. Calculated as `points_count x vectors_per_point`. Where `vectors_per_point` is a number of named vectors in schema.",
163+
description="DEPRECATED: Approximate number of vectors in collection. All vectors in collection are available for querying. Calculated as `points_count x vectors_per_point`. Where `vectors_per_point` is a number of named vectors in schema.",
157164
)
158165
indexed_vectors_count: Optional[int] = Field(
159166
default=None,
@@ -220,6 +227,7 @@ def __str__(self) -> str:
220227

221228
GREEN = "green"
222229
YELLOW = "yellow"
230+
GREY = "grey"
223231
RED = "red"
224232

225233

@@ -400,6 +408,18 @@ class CreateShardingKeyOperation(BaseModel, extra="forbid"):
400408
create_sharding_key: "CreateShardingKey" = Field(..., description="")
401409

402410

411+
class Datatype(str, Enum):
412+
"""
413+
Defines which datatype should be used to represent vectors in the storage. Choosing different datatypes allows to optimize memory usage and performance vs accuracy. - For `float32` datatype - vectors are stored as single-precision floating point numbers, 4bytes. - For `uint8` datatype - vectors are stored as unsigned 8-bit integers, 1byte. It expects vector elements to be in range `[0, 255]`.
414+
"""
415+
416+
def __str__(self) -> str:
417+
return str(self.value)
418+
419+
FLOAT32 = "float32"
420+
UINT8 = "uint8"
421+
422+
403423
class DatetimeRange(BaseModel, extra="forbid"):
404424
"""
405425
Range filter request
@@ -1018,6 +1038,10 @@ class MoveShardOperation(BaseModel, extra="forbid"):
10181038
move_shard: "MoveShard" = Field(..., description="")
10191039

10201040

1041+
class MultiVectorConfigOneOf(BaseModel):
1042+
max_sim: Any = Field(..., description="")
1043+
1044+
10211045
class NamedSparseVector(BaseModel, extra="forbid"):
10221046
"""
10231047
Sparse vector data with name
@@ -1029,7 +1053,7 @@ class NamedSparseVector(BaseModel, extra="forbid"):
10291053

10301054
class NamedVector(BaseModel, extra="forbid"):
10311055
"""
1032-
Vector data with name
1056+
Dense vector data with name
10331057
"""
10341058

10351059
name: str = Field(..., description="Name of vector data")
@@ -1822,6 +1846,17 @@ def __str__(self) -> str:
18221846
SNAPSHOT = "snapshot"
18231847

18241848

1849+
class ShardTransferMethodOneOf2(str, Enum):
1850+
"""
1851+
Attempt to transfer shard difference by WAL delta.
1852+
"""
1853+
1854+
def __str__(self) -> str:
1855+
return str(self.value)
1856+
1857+
WAL_DELTA = "wal_delta"
1858+
1859+
18251860
class ShardingMethod(str, Enum):
18261861
AUTO = "auto"
18271862
CUSTOM = "custom"
@@ -2098,6 +2133,12 @@ class VectorDataConfig(BaseModel):
20982133
quantization_config: Optional["QuantizationConfig"] = Field(
20992134
default=None, description="Vector specific quantization config that overrides collection config"
21002135
)
2136+
multi_vec_config: Optional["MultiVectorConfig"] = Field(
2137+
default=None, description="Vector specific configuration to enable multiple vectors per point"
2138+
)
2139+
datatype: Optional["VectorStorageDatatype"] = Field(
2140+
default=None, description="Vector specific configuration to set specific storage element type"
2141+
)
21012142

21022143

21032144
class VectorDataInfo(BaseModel):
@@ -2138,6 +2179,7 @@ class VectorParams(BaseModel, extra="forbid"):
21382179
default=None,
21392180
description="If true, vectors are served from disk, improving RAM usage at the cost of latency Default: false",
21402181
)
2182+
datatype: Optional["Datatype"] = Field(default=None, description="Params of single vector data storage")
21412183

21422184

21432185
class VectorParamsDiff(BaseModel, extra="forbid"):
@@ -2152,6 +2194,18 @@ class VectorParamsDiff(BaseModel, extra="forbid"):
21522194
)
21532195

21542196

2197+
class VectorStorageDatatype(str, Enum):
2198+
"""
2199+
Storage types for vectors
2200+
"""
2201+
2202+
def __str__(self) -> str:
2203+
return str(self.value)
2204+
2205+
FLOAT32 = "float32"
2206+
UINT8 = "uint8"
2207+
2208+
21552209
class VectorStorageTypeOneOf(str, Enum):
21562210
"""
21572211
Storage in memory (RAM) Will be very fast at the cost of consuming a lot of memory.
@@ -2287,6 +2341,9 @@ def __str__(self) -> str:
22872341
MatchAny,
22882342
MatchExcept,
22892343
]
2344+
MultiVectorConfig = Union[
2345+
MultiVectorConfigOneOf,
2346+
]
22902347
NamedVectorStruct = Union[
22912348
List[StrictFloat],
22922349
NamedVector,
@@ -2349,6 +2406,7 @@ def __str__(self) -> str:
23492406
ShardTransferMethod = Union[
23502407
ShardTransferMethodOneOf,
23512408
ShardTransferMethodOneOf1,
2409+
ShardTransferMethodOneOf2,
23522410
]
23532411
SparseIndexType = Union[
23542412
SparseIndexTypeOneOf,

‎qdrant_client/proto/collections.proto‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,19 @@ syntax = "proto3";
22
package qdrant;
33
option csharp_namespace = "Qdrant.Client.Grpc";
44

5+
enum Datatype {
6+
Default = 0;
7+
Float32 = 1;
8+
Uint8 = 2;
9+
}
10+
511
message VectorParams {
612
uint64 size = 1; // Size of the vectors
713
Distance distance = 2; // Distance function used for comparing vectors
814
optional HnswConfigDiff hnsw_config = 3; // Configuration of vector HNSW graph. If omitted - the collection configuration will be used
915
optional QuantizationConfig quantization_config = 4; // Configuration of vector quantization config. If omitted - the collection configuration will be used
1016
optional bool on_disk = 5; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM.
17+
optional Datatype datatype = 6; // Data type of the vectors
1118
}
1219

1320
message VectorParamsDiff {
@@ -93,6 +100,7 @@ enum CollectionStatus {
93100
Green = 1; // All segments are ready
94101
Yellow = 2; // Optimization in process
95102
Red = 3; // Something went wrong
103+
Grey = 4; // Optimization is pending
96104
}
97105

98106
enum PayloadSchemaType {
@@ -448,9 +456,8 @@ enum ReplicaState {
448456
Partial = 2; // The shard is partially loaded and is currently receiving data from other shards
449457
Initializing = 3; // Collection is being created
450458
Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
451-
PartialSnapshot = 5; // Snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
459+
PartialSnapshot = 5; // Deprecated: snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
452460
Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
453-
// TODO(1.9): deprecate PartialSnapshot state
454461
}
455462

456463
message ShardKey {
@@ -496,6 +503,12 @@ message MoveShard {
496503
optional ShardTransferMethod method = 4;
497504
}
498505

506+
message AbortShardTransfer {
507+
uint32 shard_id = 1; // Local shard id
508+
uint64 from_peer_id = 2;
509+
uint64 to_peer_id = 3;
510+
}
511+
499512
message RestartTransfer {
500513
uint32 shard_id = 1; // Local shard id
501514
uint64 from_peer_id = 2;
@@ -530,7 +543,7 @@ message UpdateCollectionClusterSetupRequest {
530543
oneof operation {
531544
MoveShard move_shard = 2;
532545
MoveShard replicate_shard = 3;
533-
MoveShard abort_transfer = 4;
546+
AbortShardTransfer abort_transfer = 4;
534547
Replica drop_replica = 5;
535548
CreateShardKey create_shard_key = 7;
536549
DeleteShardKey delete_shard_key = 8;

‎tests/conversions/fixtures.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@
163163
ef_construct=1000,
164164
),
165165
on_disk=True,
166+
datatype=grpc.Datatype.Float32,
166167
)
167168

168169
product_quantizations = [
@@ -190,6 +191,7 @@
190191
size=100,
191192
distance=grpc.Distance.Cosine,
192193
quantization_config=grpc.QuantizationConfig(scalar=scalar_quantization),
194+
datatype=grpc.Datatype.Uint8,
193195
)
194196

195197
single_vector_config = grpc.VectorsConfig(params=vector_param)
@@ -302,6 +304,7 @@
302304
collection_status = grpc.CollectionStatus.Yellow
303305
collection_status_green = grpc.CollectionStatus.Green
304306
collection_status_error = grpc.CollectionStatus.Red
307+
collection_status_grey = grpc.CollectionStatus.Grey
305308

306309
optimizer_status = grpc.OptimizerStatus(ok=True)
307310
optimizer_status_error = grpc.OptimizerStatus(ok=False, error="Error!")
@@ -359,6 +362,16 @@
359362
points=0,
360363
)
361364

365+
collection_info_grey = grpc.CollectionInfo(
366+
status=collection_status_grey,
367+
optimizer_status=optimizer_status_error,
368+
# vectors_count=100000,
369+
points_count=100000,
370+
segments_count=6,
371+
config=collection_config,
372+
payload_schema={},
373+
)
374+
362375
collection_info_ok = grpc.CollectionInfo(
363376
status=collection_status_green,
364377
optimizer_status=optimizer_status,
@@ -888,7 +901,12 @@
888901
"ValuesCount": [values_count],
889902
"Filter": [filter_nested, filter_],
890903
"OptimizersConfigDiff": [optimizer_config, optimizer_config_half],
891-
"CollectionInfo": [collection_info, collection_info_ok, collection_info_red],
904+
"CollectionInfo": [
905+
collection_info,
906+
collection_info_ok,
907+
collection_info_red,
908+
collection_info_grey,
909+
],
892910
"FieldCondition": [
893911
field_condition_match,
894912
field_condition_range,

‎tools/generate_grpc_client.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ set -e
44

55
PROJECT_ROOT="$(pwd)/$(dirname "$0")/../"
66

7+
pip install grpcio==1.48.2
8+
pip install grpcio-tools==1.48.2
9+
710
cd $(mktemp -d)
811

912
git clone --sparse --filter=blob:none --depth=1 git@github.com:qdrant/qdrant.git

0 commit comments

Comments
 (0)