Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' into v0.25-join

Ozan Armağan3cc59ecce461
diff --cc test/collection_vector_search_test.cpp
index 390b9fa7,3ffe288e..2f55cd34
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@@ -1033,114 -1033,72 +1033,184 @@@ TEST_F(CollectionVectorTest, EmbedFromO
ASSERT_TRUE(add_op.ok());
}
+ TEST_F(CollectionVectorTest, SkipEmbeddingOpWhenValueExists) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ nlohmann::json model_config = R"({
+ "model_name": "ts/e5-small"
+ })"_json;
+
+ // will be roughly 0.1110895648598671,-0.11710234731435776,-0.5319093465805054, ...
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ // document with explicit embedding vector
+ nlohmann::json doc;
+ doc["name"] = "FOO";
+
+ std::vector<float> vec;
+ for(size_t i = 0; i < 384; i++) {
+ vec.push_back(0.345);
+ }
+
+ doc["embedding"] = vec;
+
+ auto add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ // get the vector back
+ auto res = coll->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD).get();
+
+ // let's check the first few vectors
+ auto stored_vec = res["hits"][0]["document"]["embedding"];
+ ASSERT_NEAR(0.345, stored_vec[0], 0.01);
+ ASSERT_NEAR(0.345, stored_vec[1], 0.01);
+ ASSERT_NEAR(0.345, stored_vec[2], 0.01);
+ ASSERT_NEAR(0.345, stored_vec[3], 0.01);
+ ASSERT_NEAR(0.345, stored_vec[4], 0.01);
+
+ // what happens when vector contains invalid value, like string
+ doc["embedding"] = "foo"; //{0.11, 0.11};
+ add_op = coll->add(doc.dump());
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `embedding` contains an invalid embedding.", add_op.error());
+
+ // when dims don't match
+ doc["embedding"] = {0.11, 0.11};
+ add_op = coll->add(doc.dump());
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `embedding` contains an invalid embedding.", add_op.error());
+
+ // invalid array value
+ doc["embedding"].clear();
+ for(size_t i = 0; i < 384; i++) {
+ doc["embedding"].push_back(0.01);
+ }
+ doc["embedding"][5] = "foo";
+ add_op = coll->add(doc.dump());
+ ASSERT_FALSE(add_op.ok());
+ ASSERT_EQ("Field `embedding` contains invalid float values.", add_op.error());
+ }
++
+TEST_F(CollectionVectorTest, SemanticSearchReturnOnlyVectorDistance) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "product_name": "moisturizer",
+ "category": "beauty"
+ })"_json.dump());
+
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll1->search("moisturizer", {"embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ // Return only vector distance
+ ASSERT_EQ(0, results["hits"][0].count("text_match_info"));
+ ASSERT_EQ(0, results["hits"][0].count("hybrid_search_info"));
+ ASSERT_EQ(1, results["hits"][0].count("vector_distance"));
+}
+
+TEST_F(CollectionVectorTest, KeywordSearchReturnOnlyTextMatchInfo) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ auto coll1 = collection_create_op.get();
+ auto add_op = coll1->add(R"({
+ "product_name": "moisturizer",
+ "category": "beauty"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll1->search("moisturizer", {"product_name"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ // Return only text match info
+ ASSERT_EQ(0, results["hits"][0].count("vector_distance"));
+ ASSERT_EQ(0, results["hits"][0].count("hybrid_search_info"));
+ ASSERT_EQ(1, results["hits"][0].count("text_match_info"));
+}
+
+TEST_F(CollectionVectorTest, HybridSearchReturnAllInfo) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "category", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ auto coll1 = collection_create_op.get();
+
+ auto add_op = coll1->add(R"({
+ "product_name": "moisturizer",
+ "category": "beauty"
+ })"_json.dump());
+ ASSERT_TRUE(add_op.ok());
+
+
+ auto results = coll1->search("moisturizer", {"product_name", "embedding"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>()).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+
+ // Return all info
+ ASSERT_EQ(1, results["hits"][0].count("vector_distance"));
+ ASSERT_EQ(1, results["hits"][0].count("text_match_info"));
+ ASSERT_EQ(1, results["hits"][0].count("hybrid_search_info"));
- }
++}