Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat(oceanbase): add advanced ANN query options (#3649) (#3758)

ZHANG Yixin273c1bf70fb2
diff --git a/camel/storages/vectordb_storages/oceanbase.py b/camel/storages/vectordb_storages/oceanbase.py
index c732d13ef..db64431ac 100644
--- a/camel/storages/vectordb_storages/oceanbase.py
+++ b/camel/storages/vectordb_storages/oceanbase.py
@@ -14,6 +14,7 @@
import json
import logging
+import math
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional
if TYPE_CHECKING:
@@ -45,8 +46,10 @@ class OceanBaseStorage(BaseVectorStorage):
password (str): Password for the user. (default: :obj:`""`)
db_name (str): Database name in OceanBase.
(default: :obj:`"test"`)
- distance (Literal["l2", "cosine"], optional): The distance metric for
- vector comparison. Options: "l2", "cosine". (default: :obj:`"l2"`)
+ distance (Literal["l2", "cosine", "inner_product",
+ "negative_inner_product"], optional): The distance metric for
+ vector comparison. Options: "l2", "cosine", "inner_product",
+ "negative_inner_product". (default: :obj:`"l2"`)
delete_table_on_del (bool, optional): Flag to determine if the
table should be deleted upon object destruction.
(default: :obj:`False`)
@@ -66,7 +69,9 @@ class OceanBaseStorage(BaseVectorStorage):
user: str = "root@test",
password: str = "",
db_name: str = "test",
- distance: Literal["l2", "cosine"] = "l2",
+ distance: Literal[
+ "l2", "cosine", "inner_product", "negative_inner_product"
+ ] = "l2",
delete_table_on_del: bool = False,
**kwargs: Any,
) -> None:
@@ -81,7 +86,9 @@ class OceanBaseStorage(BaseVectorStorage):
self.vector_dim: int = vector_dim
self.table_name: str = table_name
- self.distance: Literal["l2", "cosine"] = distance
+ self.distance: Literal[
+ "l2", "cosine", "inner_product", "negative_inner_product"
+ ] = distance
self.delete_table_on_del: bool = delete_table_on_del
# Create client
@@ -91,8 +98,10 @@ class OceanBaseStorage(BaseVectorStorage):
# Map distance to distance function in OceanBase
self._distance_func_map: Dict[str, str] = {
- "cosine": "cosine_distance",
"l2": "l2_distance",
+ "cosine": "cosine_distance",
+ "inner_product": "inner_product",
+ "negative_inner_product": "negative_inner_product",
}
# Check or create table with vector index
@@ -281,6 +290,7 @@ class OceanBaseStorage(BaseVectorStorage):
def query(
self,
query: VectorDBQuery,
+ where_clause: Optional[List[Any]] = None,
**kwargs: Any,
) -> List[VectorDBQueryResult]:
r"""Searches for similar vectors in the storage based on the
@@ -289,6 +299,12 @@ class OceanBaseStorage(BaseVectorStorage):
Args:
query (VectorDBQuery): The query object containing the search
vector and the number of top similar vectors to retrieve.
+ where_clause (Optional[List[Any]], optional): A list of SQLAlchemy
+ expressions to filter the query results. These are passed
+ directly to pyobvector's ann_search(). Use sqlalchemy.text()
+ for raw SQL conditions. Example:
+ ``[text("metadata->>'$.category' = 'electronics'")]``.
+ (default: :obj:`None`)
**kwargs (Any): Additional keyword arguments.
Returns:
@@ -303,11 +319,13 @@ class OceanBaseStorage(BaseVectorStorage):
from sqlalchemy import func
try:
- # Get distance function name
- distance_func_name: str = self._distance_func_map.get(
+ # For inner_product, use negative_inner_product in search so
+ # ascending order returns most similar first
+ distance_func_name = self._distance_func_map.get(
self.distance, "l2_distance"
)
-
+ if self.distance == "inner_product":
+ distance_func_name = "negative_inner_product"
distance_func = getattr(func, distance_func_name)
# Validate query vector dimensions
@@ -325,6 +343,7 @@ class OceanBaseStorage(BaseVectorStorage):
with_dist=True,
topk=query.top_k,
output_column_names=["id", "embedding", "metadata"],
+ where_clause=where_clause,
)
# Convert results to VectorDBQueryResult format
@@ -418,22 +437,60 @@ class OceanBaseStorage(BaseVectorStorage):
logger.error(error_msg)
raise RuntimeError(error_msg)
+ def _stable_sigmoid(self, x: float) -> float:
+ r"""Computes sigmoid function in a numerically stable way.
+
+ Uses the identity: sigmoid(x) = exp(x) / (1 + exp(x)) for x < 0
+ and sigmoid(x) = 1 / (1 + exp(-x)) for x >= 0, to avoid overflow.
+
+ Args:
+ x (float): The input value.
+
+ Returns:
+ float: The sigmoid of x, in the range (0, 1).
+ """
+ if x >= 0:
+ return 1.0 / (1.0 + math.exp(-x))
+ else:
+ exp_x = math.exp(x)
+ return exp_x / (1.0 + exp_x)
+
def _convert_distance_to_similarity(self, distance: float) -> float:
- r"""Converts distance to similarity score based on distance metric."""
- # Ensure distance is non-negative
- distance = max(0.0, distance)
+ r"""Converts distance/score to similarity score based on distance
+ metric.
+
+ For L2 and cosine distance, lower values mean more similar, so we
+ clamp negative distances to 0 before conversion.
+ For inner product metrics, the raw score can be negative (indicating
+ opposite directions), so we do NOT clamp the distance and use sigmoid
+ to map the full range to [0, 1].
+
+ Args:
+ distance (float): The distance or score value from the query.
+
+ Returns:
+ float: A similarity score, typically in the range [0, 1].
+ """
if self.distance == "cosine":
- # Cosine distance = 1 - cosine similarity
- # Ensure similarity is between 0 and 1
+ # Cosine distance = 1 - cosine similarity, range [0, 2]
+ # Clamp to non-negative before conversion
+ distance = max(0.0, distance)
return max(0.0, min(1.0, 1.0 - distance))
elif self.distance == "l2":
- import math
-
- # Exponential decay function for L2 distance
+ # Exponential decay for L2 distance
+ # Clamp to non-negative before conversion
+ distance = max(0.0, distance)
return math.exp(-distance)
+ elif self.distance == "inner_product":
+ # Search uses negative_inner_product, so distance = -IP
+ return self._stable_sigmoid(-distance)
+ elif self.distance == "negative_inner_product":
+ # distance = -IP, similarity = sigmoid(IP)
+ return self._stable_sigmoid(-distance)
else:
- # Default normalization, ensure result is between 0 and 1
+ # Default: treat as distance where lower = more similar
+ distance = max(0.0, distance)
return max(0.0, min(1.0, 1.0 - min(1.0, distance)))
def clear(self) -> None:
diff --git a/examples/storages/oceanbase_vector_storage.py b/examples/storages/oceanbase_vector_storage.py
index 0b8edf48e..44811c2fe 100644
--- a/examples/storages/oceanbase_vector_storage.py
+++ b/examples/storages/oceanbase_vector_storage.py
@@ -14,6 +14,8 @@
import random
+from sqlalchemy import text
+
from camel.storages.vectordb_storages import (
OceanBaseStorage,
VectorDBQuery,
@@ -114,11 +116,224 @@ def main():
print(f"Vector count after clearing: {status.vector_count}")
+def example_filtered_query():
+ """Example demonstrating filtered ANN query using where_clause.
+
+ This example shows how to use the where_clause parameter to filter
+ results based on metadata using SQLAlchemy expressions.
+ """
+ print("\n=== Filtered ANN Query Example ===")
+
+ # Create storage with cosine distance
+ ob_storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="filtered_query_example",
+ uri=OB_URI,
+ user=OB_USER,
+ password=OB_PASSWORD,
+ db_name=OB_DB_NAME,
+ distance="cosine",
+ delete_table_on_del=True, # Auto-cleanup
+ )
+
+ # Add vectors with different categories in metadata
+ random.seed(42)
+ records = []
+ categories = ["electronics", "clothing", "books", "electronics", "books"]
+ prices = [199, 49, 25, 599, 15]
+
+ for i, (category, price) in enumerate(zip(categories, prices)):
+ records.append(
+ VectorRecord(
+ vector=[random.uniform(-1, 1) for _ in range(4)],
+ payload={
+ "item_id": i,
+ "category": category,
+ "price": price,
+ },
+ )
+ )
+
+ ob_storage.add(records)
+ print(f"Added {len(records)} records with categories: {categories}")
+
+ # Query without filter - returns all matching results
+ query_vector = records[0].vector
+ print("\n1. Query WITHOUT filter:")
+ results = ob_storage.query(
+ VectorDBQuery(query_vector=query_vector, top_k=5)
+ )
+ for r in results:
+ print(f" - {r.record.payload}")
+
+ # Query with category filter using where_clause
+ print("\n2. Query WITH category filter (electronics only):")
+ results = ob_storage.query(
+ VectorDBQuery(query_vector=query_vector, top_k=5),
+ where_clause=[text("metadata->>'$.category' = 'electronics'")],
+ )
+ for r in results:
+ print(f" - {r.record.payload}")
+
+ # Query with price filter
+ print("\n3. Query WITH price filter (price < 100):")
+ results = ob_storage.query(
+ VectorDBQuery(query_vector=query_vector, top_k=5),
+ where_clause=[text("CAST(metadata->>'$.price' AS SIGNED) < 100")],
+ )
+ for r in results:
+ print(f" - {r.record.payload}")
+
+ # Query with multiple conditions (AND)
+ print("\n4. Query WITH multiple conditions (books AND price<30):")
+ results = ob_storage.query(
+ VectorDBQuery(query_vector=query_vector, top_k=5),
+ where_clause=[
+ text("metadata->>'$.category' = 'books'"),
+ text("CAST(metadata->>'$.price' AS SIGNED) < 30"),
+ ],
+ )
+ for r in results:
+ print(f" - {r.record.payload}")
+
+ print("\nFiltered query example completed.")
+
+
+def example_distance_metrics():
+ """Example demonstrating different distance metrics.
+
+ OceanBase supports four distance metrics:
+ - l2: Euclidean distance (lower = more similar)
+ - cosine: Cosine distance (lower = more similar)
+ - inner_product: Inner product (higher = more similar)
+ - negative_inner_product: Negative inner product (lower = more similar)
+ """
+ print("\n=== Distance Metrics Example ===")
+
+ random.seed(123)
+ test_vector = [0.5, 0.5, 0.5, 0.5]
+
+ for distance_metric in [
+ "l2",
+ "cosine",
+ "inner_product",
+ "negative_inner_product",
+ ]:
+ print(f"\n--- Distance metric: {distance_metric} ---")
+
+ ob_storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name=f"distance_example_{distance_metric}",
+ uri=OB_URI,
+ user=OB_USER,
+ password=OB_PASSWORD,
+ db_name=OB_DB_NAME,
+ distance=distance_metric,
+ delete_table_on_del=True,
+ )
+
+ # Add some test vectors
+ records = [
+ VectorRecord(
+ vector=[1.0, 0.0, 0.0, 0.0], payload={"name": "unit_x"}
+ ),
+ VectorRecord(
+ vector=[0.0, 1.0, 0.0, 0.0], payload={"name": "unit_y"}
+ ),
+ VectorRecord(
+ vector=[0.5, 0.5, 0.5, 0.5], payload={"name": "same_as_query"}
+ ),
+ VectorRecord(
+ vector=[-0.5, -0.5, -0.5, -0.5], payload={"name": "opposite"}
+ ),
+ ]
+ ob_storage.add(records)
+
+ # Query and show results
+ results = ob_storage.query(
+ VectorDBQuery(query_vector=test_vector, top_k=4)
+ )
+
+ print(f" Query vector: {test_vector}")
+ print(" Results (ordered by similarity):")
+ for i, r in enumerate(results):
+ name = r.record.payload['name']
+ print(f" {i + 1}. {name}: sim={r.similarity:.4f}")
+
+ print("\nDistance metrics example completed.")
+
+
+def example_arbitrary_distance_metrics():
+ """Example with arbitrary vector values to demonstrate distance metrics."""
+ print("\n=== Arbitrary Distance Metrics Example ===")
+
+ # Arbitrary query vector
+ query = [0.3, -0.7, 0.2, 0.8]
+
+ # Various test vectors with different relationships to query
+ records = [
+ VectorRecord(
+ vector=[0.3, -0.7, 0.2, 0.8], payload={"name": "exact_match"}
+ ),
+ VectorRecord(
+ vector=[0.6, -1.4, 0.4, 1.6], payload={"name": "scaled_2x"}
+ ),
+ VectorRecord(
+ vector=[0.1, -0.3, 0.1, 0.4], payload={"name": "similar_dir"}
+ ),
+ VectorRecord(
+ vector=[-0.3, 0.7, -0.2, -0.8], payload={"name": "opposite"}
+ ),
+ VectorRecord(
+ vector=[0.8, 0.2, -0.5, 0.1], payload={"name": "orthogonal"}
+ ),
+ VectorRecord(vector=[0.0, 0.0, 0.0, 1.0], payload={"name": "unit_w"}),
+ ]
+
+ for distance_metric in [
+ "inner_product",
+ "negative_inner_product",
+ "cosine",
+ "l2",
+ ]:
+ print(f"\n--- Distance metric: {distance_metric} ---")
+
+ ob_storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name=f"arbitrary_example_{distance_metric}",
+ uri=OB_URI,
+ user=OB_USER,
+ password=OB_PASSWORD,
+ db_name=OB_DB_NAME,
+ distance=distance_metric,
+ delete_table_on_del=True,
+ )
+
+ ob_storage.add(records)
+
+ results = ob_storage.query(VectorDBQuery(query_vector=query, top_k=6))
+
+ print(f" Query: {query}")
+ print(" Results:")
+ for i, r in enumerate(results):
+ name = r.record.payload['name']
+ vec = r.record.vector
+ print(f" {i + 1}. {name}: sim={r.similarity:.4f}, vec={vec}")
+
+ print("\nArbitrary distance metrics example completed.")
+
+
if __name__ == "__main__":
main()
+ # Uncomment to run additional examples:
+ # example_filtered_query()
+ # example_distance_metrics()
+ # example_arbitrary_distance_metrics()
'''
===============================================================================
+Example output from main():
+
Vector dimension: 4
Initial vector count: 0
@@ -127,27 +342,141 @@ Vector count after adding batch: 1000
Querying similar vectors...
Result 1:
- ID: f33f008d-688a-468a-9a2d-e005d27ad9d8
+ ID: e8de28c7-4ec1-4fba-b71c-b9640d11c2d7
Payload: {'idx': 496, 'batch': 'example'}
Similarity: 0.9847431876706196
Result 2:
- ID: 0946ca4a-9129-4343-b339-b2f13a64c827
+ ID: 62755df8-8f8e-41db-87bc-206f54d324a6
Payload: {'idx': 306, 'batch': 'example'}
Similarity: 0.9598140307734809
Result 3:
- ID: e2505cc3-7bee-433e-a723-c830b9404f61
+ ID: 9a9dec55-cfc0-4323-a874-eb38269dc965
Payload: {'idx': 654, 'batch': 'example'}
Similarity: 0.9548076959468635
Result 4:
- ID: 8d6359ff-49fd-4fb7-99fd-bf4ca0a0cdcc
+ ID: fd579792-dba1-4aab-8338-8b504f1b5419
Payload: {'idx': 914, 'batch': 'example'}
Similarity: 0.9496165658207311
Result 5:
- ID: 45cef390-3e1f-4e16-8375-b4108951a93b
+ ID: 76a4f991-c5a7-4a68-9d40-d28311f18f32
Payload: {'idx': 806, 'batch': 'example'}
Similarity: 0.9485824622759768
Clearing vectors...
Vector count after clearing: 0
+
+-------------------------------------------------------------------------------
+Example output from example_filtered_query():
+
+=== Filtered ANN Query Example ===
+Added 5 records with categories: ['electronics', 'clothing', 'books', ...]
+
+1. Query WITHOUT filter:
+ - {'price': 199, 'item_id': 0, 'category': 'electronics'}
+ - {'price': 25, 'item_id': 2, 'category': 'books'}
+ - {'price': 599, 'item_id': 3, 'category': 'electronics'}
+ - {'price': 15, 'item_id': 4, 'category': 'books'}
+ - {'price': 49, 'item_id': 1, 'category': 'clothing'}
+
+2. Query WITH category filter (electronics only):
+ - {'price': 199, 'item_id': 0, 'category': 'electronics'}
+ - {'price': 599, 'item_id': 3, 'category': 'electronics'}
+
+3. Query WITH price filter (price < 100):
+ - {'price': 25, 'item_id': 2, 'category': 'books'}
+ - {'price': 15, 'item_id': 4, 'category': 'books'}
+ - {'price': 49, 'item_id': 1, 'category': 'clothing'}
+
+4. Query WITH multiple conditions (books AND price<30):
+ - {'price': 25, 'item_id': 2, 'category': 'books'}
+ - {'price': 15, 'item_id': 4, 'category': 'books'}
+
+Filtered query example completed.
+
+-------------------------------------------------------------------------------
+Example output from example_distance_metrics():
+
+=== Distance Metrics Example ===
+
+--- Distance metric: l2 ---
+ Query vector: [0.5, 0.5, 0.5, 0.5]
+ Results (ordered by similarity):
+ 1. same_as_query: sim=1.0000
+ 2. unit_x: sim=0.3679
+ 3. unit_y: sim=0.3679
+ 4. opposite: sim=0.1353
+
+--- Distance metric: cosine ---
+ Query vector: [0.5, 0.5, 0.5, 0.5]
+ Results (ordered by similarity):
+ 1. same_as_query: sim=1.0000
+ 2. unit_x: sim=0.5000
+ 3. unit_y: sim=0.5000
+ 4. opposite: sim=0.0000
+
+--- Distance metric: inner_product ---
+ Query vector: [0.5, 0.5, 0.5, 0.5]
+ Results (ordered by similarity):
+ 1. same_as_query: sim=0.7311
+ 2. unit_x: sim=0.6225
+ 3. unit_y: sim=0.6225
+ 4. opposite: sim=0.2689
+
+--- Distance metric: negative_inner_product ---
+ Query vector: [0.5, 0.5, 0.5, 0.5]
+ Results (ordered by similarity):
+ 1. same_as_query: sim=0.7311
+ 2. unit_x: sim=0.6225
+ 3. unit_y: sim=0.6225
+ 4. opposite: sim=0.2689
+
+Distance metrics example completed.
+
+-------------------------------------------------------------------------------
+Example output from example_arbitrary_distance_metrics():
+
+=== Arbitrary Distance Metrics Example ===
+
+--- Distance metric: inner_product ---
+ Query: [0.3, -0.7, 0.2, 0.8]
+ Results:
+ 1. scaled_2x: sim=0.9255, vec=[0.6, -1.4, 0.4, 1.6]
+ 2. exact_match: sim=0.7790, vec=[0.3, -0.7, 0.2, 0.8]
+ 3. unit_w: sim=0.6900, vec=[0.0, 0.0, 0.0, 1.0]
+ 4. similar_dir: sim=0.6411, vec=[0.1, -0.3, 0.1, 0.4]
+ 5. orthogonal: sim=0.5200, vec=[0.8, 0.2, -0.5, 0.1]
+ 6. opposite: sim=0.2210, vec=[-0.3, 0.7, -0.2, -0.8]
+
+--- Distance metric: negative_inner_product ---
+ Query: [0.3, -0.7, 0.2, 0.8]
+ Results:
+ 1. scaled_2x: sim=0.9255, vec=[0.6, -1.4, 0.4, 1.6]
+ 2. exact_match: sim=0.7790, vec=[0.3, -0.7, 0.2, 0.8]
+ 3. unit_w: sim=0.6900, vec=[0.0, 0.0, 0.0, 1.0]
+ 4. similar_dir: sim=0.6411, vec=[0.1, -0.3, 0.1, 0.4]
+ 5. orthogonal: sim=0.5200, vec=[0.8, 0.2, -0.5, 0.1]
+ 6. opposite: sim=0.2210, vec=[-0.3, 0.7, -0.2, -0.8]
+
+--- Distance metric: cosine ---
+ Query: [0.3, -0.7, 0.2, 0.8]
+ Results:
+ 1. scaled_2x: sim=1.0000, vec=[0.6, -1.4, 0.4, 1.6]
+ 2. exact_match: sim=1.0000, vec=[0.3, -0.7, 0.2, 0.8]
+ 3. similar_dir: sim=0.9944, vec=[0.1, -0.3, 0.1, 0.4]
+ 4. unit_w: sim=0.7127, vec=[0.0, 0.0, 0.0, 1.0]
+ 5. orthogonal: sim=0.0735, vec=[0.8, 0.2, -0.5, 0.1]
+ 6. opposite: sim=0.0000, vec=[-0.3, 0.7, -0.2, -0.8]
+
+--- Distance metric: l2 ---
+ Query: [0.3, -0.7, 0.2, 0.8]
+ Results:
+ 1. exact_match: sim=1.0000, vec=[0.3, -0.7, 0.2, 0.8]
+ 2. similar_dir: sim=0.5443, vec=[0.1, -0.3, 0.1, 0.4]
+ 3. unit_w: sim=0.4438, vec=[0.0, 0.0, 0.0, 1.0]
+ 4. scaled_2x: sim=0.3255, vec=[0.6, -1.4, 0.4, 1.6]
+ 5. orthogonal: sim=0.2397, vec=[0.8, 0.2, -0.5, 0.1]
+ 6. opposite: sim=0.1059, vec=[-0.3, 0.7, -0.2, -0.8]
+
+Arbitrary distance metrics example completed.
===============================================================================
'''
diff --git a/test/storages/vector_storages/test_oceanbase.py b/test/storages/vector_storages/test_oceanbase.py
index 04c7c5184..56fae0e46 100644
--- a/test/storages/vector_storages/test_oceanbase.py
+++ b/test/storages/vector_storages/test_oceanbase.py
@@ -245,3 +245,280 @@ def test_distance_to_similarity_conversion():
)
l2_sim = l2_storage._convert_distance_to_similarity(0.2)
assert l2_sim == pytest.approx(math.exp(-0.2))
+
+
+def test_stable_sigmoid_overflow():
+ """Test that _stable_sigmoid handles large values without overflow."""
+
+ def create_mock_index_params():
+ mock_index_param = MagicMock()
+ mock_index_params = MagicMock()
+ mock_index_params.__iter__.return_value = iter([mock_index_param])
+ return mock_index_params
+
+ with patch('pyobvector.client.ObVecClient'):
+ with patch('pyobvector.schema.VECTOR'):
+ with patch(
+ 'pyobvector.client.index_param.IndexParams',
+ side_effect=create_mock_index_params,
+ ):
+ with patch('pyobvector.client.index_param.IndexParam'):
+ storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="test_table",
+ distance="inner_product",
+ )
+
+ # Test large positive value (should approach 1.0)
+ large_positive = storage._stable_sigmoid(1000.0)
+ assert large_positive == pytest.approx(1.0)
+ assert large_positive <= 1.0
+
+ # Test large negative value (should approach 0.0)
+ large_negative = storage._stable_sigmoid(-1000.0)
+ assert large_negative == pytest.approx(0.0)
+ assert large_negative >= 0.0
+
+ # Ensure no OverflowError is raised
+ try:
+ storage._stable_sigmoid(10000.0)
+ storage._stable_sigmoid(-10000.0)
+ except OverflowError:
+ pytest.fail("_stable_sigmoid raised OverflowError")
+
+
+def test_inner_product_distance_metrics():
+ """Test initialization and similarity conversion for inner product
+ distance metrics.
+
+ Note: For inner_product distance metric, the search uses
+ negative_inner_product function internally, so
+ _convert_distance_to_similarity receives neg_ip = -IP and computes
+ sigmoid(-neg_ip) = sigmoid(IP). This means positive input
+ (neg_ip > 0) -> low similarity, and negative input
+ (neg_ip < 0, i.e., high IP) -> high similarity.
+ """
+ import math
+
+ def create_mock_index_params():
+ mock_index_param = MagicMock()
+ mock_index_params = MagicMock()
+ mock_index_params.__iter__.return_value = iter([mock_index_param])
+ return mock_index_params
+
+ with patch('pyobvector.client.ObVecClient'):
+ with patch('pyobvector.schema.VECTOR'):
+ with patch(
+ 'pyobvector.client.index_param.IndexParams',
+ side_effect=create_mock_index_params,
+ ):
+ with patch('pyobvector.client.index_param.IndexParam'):
+ # Test inner_product distance metric
+ ip_storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="test_table",
+ distance="inner_product",
+ )
+ assert ip_storage.distance == "inner_product"
+
+ # For inner_product, input is neg_ip = -IP
+ # similarity = sigmoid(-neg_ip) = sigmoid(IP)
+ # When neg_ip = 0 (IP = 0): sigmoid(0) = 0.5
+ ip_sim_zero = ip_storage._convert_distance_to_similarity(
+ 0.0
+ )
+ assert ip_sim_zero == pytest.approx(0.5)
+
+ # neg_ip = -2.0 means IP = 2.0 (high similarity)
+ # similarity = sigmoid(2.0) > 0.5
+ ip_sim_positive = (
+ ip_storage._convert_distance_to_similarity(-2.0)
+ )
+ expected_positive = 1.0 / (1.0 + math.exp(-2.0))
+ assert ip_sim_positive == pytest.approx(expected_positive)
+ assert ip_sim_positive > 0.5
+
+ # neg_ip = 2.0 means IP = -2.0 (low similarity)
+ # similarity = sigmoid(-2.0) < 0.5
+ ip_sim_negative = (
+ ip_storage._convert_distance_to_similarity(2.0)
+ )
+ expected_negative = 1.0 / (1.0 + math.exp(2.0))
+ assert ip_sim_negative == pytest.approx(expected_negative)
+ assert ip_sim_negative < 0.5
+
+ # Test negative_inner_product distance metric
+ # For negative_inner_product, search also uses
+ # negative_inner_product, so input is neg_ip = -IP
+ nip_storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="test_table",
+ distance="negative_inner_product",
+ )
+ assert nip_storage.distance == "negative_inner_product"
+
+ # similarity = sigmoid(-neg_ip) = sigmoid(IP)
+ # When neg_ip = 0: sigmoid(0) = 0.5
+ nip_sim_zero = nip_storage._convert_distance_to_similarity(
+ 0.0
+ )
+ assert nip_sim_zero == pytest.approx(0.5)
+
+ # neg_ip = 2.0 means IP = -2.0 (low similarity)
+ # similarity = sigmoid(-2.0) < 0.5
+ nip_sim_positive = (
+ nip_storage._convert_distance_to_similarity(2.0)
+ )
+ expected_nip_positive = 1.0 / (1.0 + math.exp(2.0))
+ assert nip_sim_positive == pytest.approx(
+ expected_nip_positive
+ )
+ assert nip_sim_positive < 0.5
+
+ # neg_ip = -2.0 means IP = 2.0 (high similarity)
+ # similarity = sigmoid(2.0) > 0.5
+ nip_sim_negative = (
+ nip_storage._convert_distance_to_similarity(-2.0)
+ )
+ expected_nip_negative = 1.0 / (1.0 + math.exp(-2.0))
+ assert nip_sim_negative == pytest.approx(
+ expected_nip_negative
+ )
+ assert nip_sim_negative > 0.5
+
+
+def test_query_with_where_clause(mock_ob_client):
+ """Test that where_clause parameter is passed to ann_search."""
+ from sqlalchemy import text
+
+ def create_mock_index_params():
+ mock_index_param = MagicMock()
+ mock_index_params = MagicMock()
+ mock_index_params.__iter__.return_value = iter([mock_index_param])
+ return mock_index_params
+
+ with patch('pyobvector.schema.VECTOR'):
+ with patch(
+ 'pyobvector.client.index_param.IndexParams',
+ side_effect=create_mock_index_params,
+ ):
+ with patch('pyobvector.client.index_param.IndexParam'):
+ with patch('sqlalchemy.func'):
+ # Setup mock for query results
+ mock_result = MagicMock()
+ mock_result._mapping = {
+ "id": 1,
+ "embedding": [0.1, 0.2, 0.3, 0.4],
+ "metadata": {"category": "electronics"},
+ "l2_distance": 0.3,
+ }
+ mock_ob_client.ann_search.return_value = [mock_result]
+
+ # Initialize storage
+ storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="test_table",
+ uri="127.0.0.1:2881",
+ )
+
+ # Test query with where_clause
+ query = VectorDBQuery(
+ query_vector=[0.1, 0.2, 0.3, 0.4], top_k=5
+ )
+ filter_clause = [
+ text("metadata->>'$.category' = 'electronics'")
+ ]
+
+ results = storage.query(query, where_clause=filter_clause)
+
+ # Verify ann_search was called with where_clause
+ call_kwargs = mock_ob_client.ann_search.call_args.kwargs
+ assert 'where_clause' in call_kwargs
+ assert call_kwargs['where_clause'] == filter_clause
+
+ # Verify results are returned
+ assert len(results) == 1
+ assert results[0].record.payload == {
+ "category": "electronics"
+ }
+
+
+def test_query_without_where_clause(mock_ob_client):
+ """Test that query works correctly without where_clause."""
+
+ def create_mock_index_params():
+ mock_index_param = MagicMock()
+ mock_index_params = MagicMock()
+ mock_index_params.__iter__.return_value = iter([mock_index_param])
+ return mock_index_params
+
+ with patch('pyobvector.schema.VECTOR'):
+ with patch(
+ 'pyobvector.client.index_param.IndexParams',
+ side_effect=create_mock_index_params,
+ ):
+ with patch('pyobvector.client.index_param.IndexParam'):
+ with patch('sqlalchemy.func'):
+ # Setup mock for query results
+ mock_result = MagicMock()
+ mock_result._mapping = {
+ "id": 1,
+ "embedding": [0.1, 0.2, 0.3, 0.4],
+ "metadata": {"text": "test"},
+ "l2_distance": 0.1,
+ }
+ mock_ob_client.ann_search.return_value = [mock_result]
+
+ # Initialize storage
+ storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="test_table",
+ uri="127.0.0.1:2881",
+ )
+
+ # Test query without where_clause
+ query = VectorDBQuery(
+ query_vector=[0.1, 0.2, 0.3, 0.4], top_k=1
+ )
+ results = storage.query(query)
+
+ # Verify ann_search was called with where_clause=None
+ call_kwargs = mock_ob_client.ann_search.call_args.kwargs
+ assert call_kwargs.get('where_clause') is None
+
+ # Verify results are returned
+ assert len(results) == 1
+
+
+ "distance",
+ ["l2", "cosine", "inner_product", "negative_inner_product"],
+)
+def test_all_distance_metrics_initialization(distance):
+ """Test initialization with all supported distance metrics."""
+
+ def create_mock_index_params():
+ mock_index_param = MagicMock()
+ mock_index_params = MagicMock()
+ mock_index_params.__iter__.return_value = iter([mock_index_param])
+ return mock_index_params
+
+ with patch('pyobvector.client.ObVecClient'):
+ with patch('pyobvector.schema.VECTOR'):
+ with patch(
+ 'pyobvector.client.index_param.IndexParams',
+ side_effect=create_mock_index_params,
+ ):
+ with patch('pyobvector.client.index_param.IndexParam'):
+ storage = OceanBaseStorage(
+ vector_dim=4,
+ table_name="test_table",
+ distance=distance,
+ )
+ assert storage.distance == distance
+
+ # Verify the distance function mapping exists
+ assert distance in storage._distance_func_map or any(
+ distance in key
+ for key in storage._distance_func_map.keys()
+ )