Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Milvus bump up to 2.4 (#482)

Wendong-Fanf49ebe41a46e
diff --git a/camel/storages/vectordb_storages/milvus.py b/camel/storages/vectordb_storages/milvus.py
index 7030838e6..3615ef359 100644
--- a/camel/storages/vectordb_storages/milvus.py
+++ b/camel/storages/vectordb_storages/milvus.py
@@ -118,40 +118,47 @@ class MilvusStorage(BaseVectorStorage):
Args:
collection_name (str): Name of the collection to be created.
- **kwargs (Any): Additional keyword arguments.
+ **kwargs (Any): Additional keyword arguments pass to create
+ collection.
"""
- from pymilvus import CollectionSchema, DataType, FieldSchema
-
- index_params = {
- "metric_type": "COSINE",
- "index_type": "AUTOINDEX",
- "params": {},
- }
- fields = [
- FieldSchema(name='id', dtype=DataType.VARCHAR,
- descrition='A unique identifier for the vector',
- is_primary=True, auto_id=False, max_length=65535),
- # limit from Milvus is 65535 https://milvus.io/docs/limitations.md
- FieldSchema(
- name='payload', dtype=DataType.JSON,
- description=('Any additional metadata or information related'
- 'to the vector')),
- FieldSchema(
- name='vector', dtype=DataType.FLOAT_VECTOR,
- description='The numerical representation of the vector',
- dim=self.vector_dim)
- ]
- schema = CollectionSchema(fields=fields,
- description='Collection schema')
-
- self._client.create_collection_with_schema(
+ from pymilvus import DataType
+
+ # Set the schema
+ schema = self._client.create_schema(auto_id=False,
+ enable_dynamic_field=True,
+ description='collection schema')
+
+ schema.add_field(field_name="id", datatype=DataType.VARCHAR,
+ descrition='A unique identifier for the vector',
+ is_primary=True, max_length=65535)
+ # max_length reference: https://milvus.io/docs/limitations.md
+ schema.add_field(
+ field_name="vector", datatype=DataType.FLOAT_VECTOR,
+ description='The numerical representation of the vector',
+ dim=self.vector_dim)
+ schema.add_field(
+ field_name="payload", datatype=DataType.JSON,
+ description=('Any additional metadata or information related'
+ 'to the vector'))
+
+ # Create the collection
+ self._client.create_collection(
collection_name=collection_name,
schema=schema,
- index_params=index_params,
**kwargs,
)
+ # Set the index of the parameters
+ index_params = self._client.prepare_index_params()
+
+ index_params.add_index(field_name="vector", metric_type="COSINE",
+ index_type="AUTOINDEX",
+ index_name="vector_index")
+
+ self._client.create_index(collection_name=collection_name,
+ index_params=index_params)
+
def _delete_collection(
self,
collection_name: str,
@@ -198,10 +205,11 @@ class MilvusStorage(BaseVectorStorage):
Dict[str, Any]: A dictionary containing details about the
collection.
"""
- vector_count = self._client.num_entities(collection_name)
-
+ vector_count = self._client.get_collection_stats(
+ collection_name)['row_count']
collection_info = self._client.describe_collection(collection_name)
collection_id = collection_info['collection_id']
+
dim_value = next(
(field['params']['dim']
for field in collection_info['fields'] if field['description'] ==
@@ -230,13 +238,10 @@ class MilvusStorage(BaseVectorStorage):
for record in records:
record_dict = {
- "id":
- record.id,
+ "id": record.id,
"payload":
- record.payload['message']
- if record.payload is not None else '',
- "vector":
- record.vector,
+ record.payload if record.payload is not None else '',
+ "vector": record.vector,
}
validated_data.append(record_dict)
@@ -251,7 +256,7 @@ class MilvusStorage(BaseVectorStorage):
Args:
records (List[VectorRecord]): List of vectors to be added.
- **kwargs (Any): Additional keyword arguments.
+ **kwargs (Any): Additional keyword arguments pass to insert.
Raises:
RuntimeError: If there was an error in the addition process.
@@ -277,7 +282,7 @@ class MilvusStorage(BaseVectorStorage):
Args:
ids (List[str]): List of unique identifiers for the vectors to be
deleted.
- **kwargs (Any): Additional keyword arguments.
+ **kwargs (Any): Additional keyword arguments passed to delete.
Raises:
RuntimeError: If there is an error during the deletion process.
@@ -313,7 +318,7 @@ class MilvusStorage(BaseVectorStorage):
Args:
query (VectorDBQuery): The query object containing the search
vector and the number of top similar vectors to retrieve.
- **kwargs (Any): Additional keyword arguments.
+ **kwargs (Any): Additional keyword arguments passed to search.
Returns:
List[VectorDBQueryResult]: A list of vectors retrieved from the
diff --git a/poetry.lock b/poetry.lock
index 1c523cfe5..546927857 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
[[package]]
name = "accelerate"
@@ -912,13 +912,13 @@ vision = ["Pillow (>=6.2.1)"]
[[package]]
name = "diffusers"
-version = "0.27.1"
+version = "0.27.2"
description = "State-of-the-art diffusion in PyTorch and JAX."
optional = true
python-versions = ">=3.8.0"
files = [
- {file = "diffusers-0.27.1-py3-none-any.whl", hash = "sha256:117822e219ba99a0285adf260db0f4700fbed51d2c0a5db84789408b3898ba8b"},
- {file = "diffusers-0.27.1.tar.gz", hash = "sha256:0c3632cfbb0ae29e22c6160cf8bc977afecdfe66eebe6a6bbb100c58742136ff"},
+ {file = "diffusers-0.27.2-py3-none-any.whl", hash = "sha256:85da5cd1098ab428535d592136973ec0c3f12f78148c94b379cb9f02d2414e75"},
+ {file = "diffusers-0.27.2.tar.gz", hash = "sha256:6cefd7770d7fc1d139614233aa17cdcd639c138d0c3517b8d8bbc8cf573050a0"},
]
[package.dependencies]
@@ -1689,13 +1689,13 @@ files = [
[[package]]
name = "importlib-metadata"
-version = "7.0.2"
+version = "7.1.0"
description = "Read metadata from Python packages"
optional = false
python-versions = ">=3.8"
files = [
- {file = "importlib_metadata-7.0.2-py3-none-any.whl", hash = "sha256:f4bc4c0c070c490abf4ce96d715f68e95923320370efb66143df00199bb6c100"},
- {file = "importlib_metadata-7.0.2.tar.gz", hash = "sha256:198f568f3230878cb1b44fbd7975f87906c22336dba2e4a7f05278c281fbd792"},
+ {file = "importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570"},
+ {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"},
]
[package.dependencies]
@@ -1704,17 +1704,17 @@ zipp = ">=0.5"
[package.extras]
docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
perf = ["ipython"]
-testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"]
+testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"]
[[package]]
name = "importlib-resources"
-version = "6.3.1"
+version = "6.4.0"
description = "Read resources from Python packages"
optional = false
python-versions = ">=3.8"
files = [
- {file = "importlib_resources-6.3.1-py3-none-any.whl", hash = "sha256:4811639ca7fa830abdb8e9ca0a104dc6ad13de691d9fe0d3173a71304f068159"},
- {file = "importlib_resources-6.3.1.tar.gz", hash = "sha256:29a3d16556e330c3c8fb8202118c5ff41241cc34cbfb25989bbad226d99b7995"},
+ {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"},
+ {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"},
]
[package.dependencies]
@@ -1722,7 +1722,7 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""}
[package.extras]
docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"]
-testing = ["jaraco.collections", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"]
+testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"]
[[package]]
name = "iniconfig"
@@ -2577,13 +2577,13 @@ wheel = "*"
[[package]]
name = "openai"
-version = "1.14.1"
+version = "1.14.2"
description = "The official Python library for the openai API"
optional = false
python-versions = ">=3.7.1"
files = [
- {file = "openai-1.14.1-py3-none-any.whl", hash = "sha256:f9322b0bf3b82bbd06930fad535369a023f35a3a96d3ef0b827644a15d7aae97"},
- {file = "openai-1.14.1.tar.gz", hash = "sha256:1fab5dd623cdc0c7c6e7da5d8d11fa6900f94191c2dfb6510d7eac33195fa175"},
+ {file = "openai-1.14.2-py3-none-any.whl", hash = "sha256:a48b3c4d635b603952189ac5a0c0c9b06c025b80eb2900396939f02bb2104ac3"},
+ {file = "openai-1.14.2.tar.gz", hash = "sha256:e5642f7c02cf21994b08477d7bb2c1e46d8f335d72c26f0396c5f89b15b5b153"},
]
[package.dependencies]
@@ -3270,13 +3270,13 @@ windows-terminal = ["colorama (>=0.4.6)"]
[[package]]
name = "pymilvus"
-version = "2.3.7"
+version = "2.4.0"
description = "Python Sdk for Milvus"
optional = true
python-versions = ">=3.8"
files = [
- {file = "pymilvus-2.3.7-py3-none-any.whl", hash = "sha256:37d5a360d671c6fe23fe1dd4e6b41af6e4b6d6488ad8e43a06afe23d02f98272"},
- {file = "pymilvus-2.3.7.tar.gz", hash = "sha256:b8df5b8db3a82209c33b7211e0b9ef4a63ee00cb2976ccb1e9f5b92a2c2d5b82"},
+ {file = "pymilvus-2.4.0-1-py3-none-any.whl", hash = "sha256:9f8212af51bc235a4c1230c344a852b6f75a4be01cac38f21dab5b65695a9598"},
+ {file = "pymilvus-2.4.0.tar.gz", hash = "sha256:f1d1a2e9d5172fea3e0a5d396bed9de561a2b93a4d3e1945fdf68b74d771fb05"},
]
[package.dependencies]
@@ -3292,6 +3292,9 @@ requests = "*"
setuptools = ">=67"
ujson = ">=2.0.0"
+[package.extras]
+model = ["milvus-model (>=0.1.0)"]
+
[[package]]
name = "pymupdf"
version = "1.23.26"
@@ -4559,13 +4562,13 @@ widechars = ["wcwidth"]
[[package]]
name = "threadpoolctl"
-version = "3.3.0"
+version = "3.4.0"
description = "threadpoolctl"
optional = true
python-versions = ">=3.8"
files = [
- {file = "threadpoolctl-3.3.0-py3-none-any.whl", hash = "sha256:6155be1f4a39f31a18ea70f94a77e0ccd57dced08122ea61109e7da89883781e"},
- {file = "threadpoolctl-3.3.0.tar.gz", hash = "sha256:5dac632b4fa2d43f42130267929af3ba01399ef4bd1882918e92dbc30365d30c"},
+ {file = "threadpoolctl-3.4.0-py3-none-any.whl", hash = "sha256:8f4c689a65b23e5ed825c8436a92b818aac005e0f3715f6a1664d7c7ee29d262"},
+ {file = "threadpoolctl-3.4.0.tar.gz", hash = "sha256:f11b491a03661d6dd7ef692dd422ab34185d982466c49c8f98c8f716b5c93196"},
]
[[package]]
@@ -4842,13 +4845,13 @@ telegram = ["requests"]
[[package]]
name = "transformers"
-version = "4.38.2"
+version = "4.39.0"
description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow"
optional = true
python-versions = ">=3.8.0"
files = [
- {file = "transformers-4.38.2-py3-none-any.whl", hash = "sha256:c4029cb9f01b3dd335e52f364c52d2b37c65b4c78e02e6a08b1919c5c928573e"},
- {file = "transformers-4.38.2.tar.gz", hash = "sha256:c5fc7ad682b8a50a48b2a4c05d4ea2de5567adb1bdd00053619dbe5960857dd5"},
+ {file = "transformers-4.39.0-py3-none-any.whl", hash = "sha256:7801785b1f016d667467e8c372c1c3653c18fe32ba97952059e3bea79ba22b08"},
+ {file = "transformers-4.39.0.tar.gz", hash = "sha256:517a13cd633b10bea01c92ab0b3059762872c7c29da3d223db9d28e926fe330d"},
]
[package.dependencies]
@@ -5222,13 +5225,13 @@ zstd = ["zstandard (>=0.18.0)"]
[[package]]
name = "uvicorn"
-version = "0.28.1"
+version = "0.29.0"
description = "The lightning-fast ASGI server."
optional = false
python-versions = ">=3.8"
files = [
- {file = "uvicorn-0.28.1-py3-none-any.whl", hash = "sha256:5162f6d652f545be91b1feeaee8180774af143965ca9dc8a47ff1dc6bafa4ad5"},
- {file = "uvicorn-0.28.1.tar.gz", hash = "sha256:08103e79d546b6cf20f67c7e5e434d2cf500a6e29b28773e407250c54fc4fa3c"},
+ {file = "uvicorn-0.29.0-py3-none-any.whl", hash = "sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de"},
+ {file = "uvicorn-0.29.0.tar.gz", hash = "sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0"},
]
[package.dependencies]
@@ -5623,4 +5626,4 @@ vector-databases = ["pymilvus", "qdrant-client"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<3.12"
-content-hash = "d0f80d9bdb8c9fa54158f229442044b9fa3f02da9be867d5e434323f94995bec"
+content-hash = "90037d7fae0c35fe0c507bc5ba1137e70cf82f31aaa919b426684fbb5a4f52a3"
diff --git a/pyproject.toml b/pyproject.toml
index 3633c5baa..6192bd504 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -61,7 +61,7 @@ sentence-transformers = { version = "^2.2.2", optional = true }
# vector-databases
qdrant-client = { version = "^1.6.4", optional = true }
-pymilvus = { version = "^2.3.7", optional = true }
+pymilvus = { version = "^2.4.0", optional = true }
# test
pytest = { version = "^7", optional = true}