Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions docs/reference/online-stores/milvus.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,36 @@ online_store:
```
{% endcode %}

Connecting to [Zilliz Cloud](https://zilliz.com/cloud) (managed Milvus) with an API key.
Read the token from an environment variable rather than committing it:

{% code title="feature_store.yaml" %}
```yaml
project: my_feature_repo
registry: data/registry.db
provider: local
online_store:
type: milvus
uri: "https://<your-cluster-endpoint>" # Public Endpoint from the Zilliz Cloud console
token: ${ZILLIZ_TOKEN} # pragma: allowlist secret
db_name: "default"
embedding_dim: 768
index_type: "AUTOINDEX"
metric_type: "COSINE"
```
{% endcode %}

## Configuration options

| Option | Default | Description |
|:-------|:--------|:------------|
| `path` | `""` | Path to a Milvus Lite database file. Used when `provider: local` and `path` is set. |
| `host` | `http://localhost` | Milvus server host, including the scheme. |
| `port` | `19530` | Milvus server port. |
| `uri` | unset | Full endpoint, e.g. `https://<cluster>.zillizcloud.com:19530`. Takes precedence over `host`/`port`, and over `path`. |
| `username` / `password` | `""` | Credentials, sent as the token `username:password`. |
| `token` | unset | API key or `username:password`. Takes precedence over `username`/`password`. |
| `db_name` | unset | Milvus database to use. The database must already exist. Defaults to the server's `default` database. |
| `embedding_dim` | `128` | Dimension of vector fields. |
| `index_type` | `FLAT` | Index type for vector fields with `vector_index=True`. |
| `metric_type` | `COSINE` | Default metric when a field does not set `vector_search_metric`. |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,12 @@ class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig):
path: Optional[StrictStr] = ""
host: Optional[StrictStr] = "http://localhost"
port: Optional[int] = 19530
# Full endpoint, e.g. a Zilliz Cloud https URI. Takes precedence over host/port.
uri: Optional[StrictStr] = None
# API key or "username:password". Takes precedence over username/password.
token: Optional[StrictStr] = None
# Milvus database to use. The database must already exist.
db_name: Optional[StrictStr] = None
index_type: Optional[str] = "FLAT"
metric_type: Optional[str] = "COSINE"
embedding_dim: Optional[int] = 128
Expand Down Expand Up @@ -258,22 +264,25 @@ def _get_db_path(self, config: RepoConfig) -> str:

def _connect(self, config: RepoConfig) -> MilvusClient:
if not self.client:
if config.provider == "local" and config.online_store.path:
online_config = config.online_store
if (
config.provider == "local"
and online_config.path
and not online_config.uri
):
db_path = self._get_db_path(config)
logger.info("Connecting to Milvus in local mode using %s", db_path)
self.client = MilvusClient(db_path)
else:
logger.info(
"Connecting to Milvus remotely at %s:%s",
config.online_store.host,
config.online_store.port,
)
self.client = MilvusClient(
uri=f"{config.online_store.host}:{config.online_store.port}",
token=f"{config.online_store.username}:{config.online_store.password}"
if config.online_store.username and config.online_store.password
else "",
)
uri = online_config.uri or f"{online_config.host}:{online_config.port}"
logger.info("Connecting to Milvus remotely at %s", uri)
client_kwargs: Dict[str, Any] = {
"uri": uri,
"token": _milvus_token(online_config),
}
if online_config.db_name:
client_kwargs["db_name"] = online_config.db_name
self.client = MilvusClient(**client_kwargs)
return self.client

def _get_or_create_collection(
Expand Down Expand Up @@ -1019,6 +1028,15 @@ def _table_id(project: str, table: FeatureView, enable_versioning: bool = False)
return compute_table_id(project, table, enable_versioning)


def _milvus_token(online_config: MilvusOnlineStoreConfig) -> str:
"""Return the token to authenticate with: ``token``, else ``username:password``."""
if online_config.token:
return online_config.token
if online_config.username and online_config.password:
return f"{online_config.username}:{online_config.password}"
return ""


def _get_composite_key_name(table: FeatureView) -> str:
return "_".join([field.name for field in table.entity_columns]) + "_pk"

Expand Down
36 changes: 24 additions & 12 deletions sdk/python/tests/integration/online_store/test_milvus_remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
ZILLIZ_URI=https://<cluster>.zillizcloud.com ZILLIZ_TOKEN=<user:password> \
pytest --integration sdk/python/tests/integration/online_store/test_milvus_remote.py

They also run against a self-hosted Milvus server, e.g.
``ZILLIZ_TOKEN`` can be an API key or ``username:password``. The tests also run
against a self-hosted Milvus server, e.g.
``ZILLIZ_URI=http://localhost:19530 ZILLIZ_TOKEN=root:Milvus``.
"""

Expand All @@ -16,7 +17,6 @@
from pathlib import Path
from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar
from unittest.mock import patch
from urllib.parse import urlparse

import pytest

Expand Down Expand Up @@ -44,16 +44,7 @@


def _connection_config() -> Dict[str, Any]:
assert ZILLIZ_URI and ZILLIZ_TOKEN
parsed = urlparse(ZILLIZ_URI)
default_port = 443 if parsed.scheme == "https" else 19530
username, _, password = ZILLIZ_TOKEN.partition(":")
return {
"host": f"{parsed.scheme}://{parsed.hostname}",
"port": parsed.port or default_port,
"username": username,
"password": password,
}
return {"uri": ZILLIZ_URI, "token": ZILLIZ_TOKEN}


def _repo_config(tmp_path: Path, project: str, **online_store: Any) -> RepoConfig:
Expand Down Expand Up @@ -277,3 +268,24 @@ def test_released_collection_is_loaded_once(

assert rows[0] is not None and rows[0]["city"].string_val == "Paris"
assert load_spy.call_count == 1


def test_db_name(tmp_path: Path, project: str, store: MilvusOnlineStore) -> None:
from pymilvus import MilvusClient

db_name = os.environ.get("ZILLIZ_DB_NAME", "feast_it_db")
admin = MilvusClient(uri=ZILLIZ_URI, token=ZILLIZ_TOKEN)
try:
if db_name not in admin.list_databases():
admin.create_database(db_name)
except Exception as e:
pytest.skip(f"Cannot create database {db_name!r} on this cluster: {e}")

config = _repo_config(tmp_path, project, db_name=db_name)
fv = _scalar_feature_view()
store.update(config, [], [fv], [], [], partial=False)

collection_name = f"{project}_{fv.name}"
db_client = MilvusClient(uri=ZILLIZ_URI, token=ZILLIZ_TOKEN, db_name=db_name)
assert collection_name in db_client.list_collections()
assert collection_name not in admin.list_collections()
Original file line number Diff line number Diff line change
Expand Up @@ -301,3 +301,77 @@ def test_load_collection_not_called_per_query(tmp_path: Path) -> None:
)

assert load_spy.call_count == 0


@pytest.mark.parametrize(
"online_store, expected_kwargs",
[
# Existing configs build exactly the same client arguments as before.
({}, {"uri": "http://localhost:19530", "token": ""}),
(
{
"host": "https://milvus.internal",
"port": 443,
"username": "u",
"password": "p",
},
{"uri": "https://milvus.internal:443", "token": "u:p"},
),
# token takes precedence over username/password.
(
{"username": "u", "password": "p", "token": "api-key"},
{"uri": "http://localhost:19530", "token": "api-key"},
),
# uri takes precedence over host/port.
(
{
"uri": "https://in01-abc.zillizcloud.com",
"host": "http://ignored",
"token": "k",
},
{"uri": "https://in01-abc.zillizcloud.com", "token": "k"},
),
(
{
"uri": "https://in01-abc.zillizcloud.com",
"token": "k",
"db_name": "catalog",
},
{
"uri": "https://in01-abc.zillizcloud.com",
"token": "k",
"db_name": "catalog",
},
),
],
)
@patch(f"{MILVUS_MODULE}.MilvusClient")
def test_remote_client_arguments(
mock_client_cls: MagicMock,
online_store: Dict[str, Any],
expected_kwargs: Dict[str, Any],
) -> None:
config = _mock_config(**online_store)
config.provider = "gcp"

MilvusOnlineStore()._connect(config)

mock_client_cls.assert_called_once_with(**expected_kwargs)


@patch(f"{MILVUS_MODULE}.MilvusClient")
def test_uri_takes_precedence_over_lite_path(mock_client_cls: MagicMock) -> None:
config = _mock_config(path="online_store.db", uri="http://milvus:19530", token="k")

MilvusOnlineStore()._connect(config)

mock_client_cls.assert_called_once_with(uri="http://milvus:19530", token="k")


@patch(f"{MILVUS_MODULE}.MilvusClient")
def test_lite_path_used_without_uri(mock_client_cls: MagicMock) -> None:
config = _mock_config(path="/tmp/online_store.db", token="ignored")

MilvusOnlineStore()._connect(config)

mock_client_cls.assert_called_once_with("/tmp/online_store.db")
Loading