Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' into v0.25-join
Ozan Armağan3cc59ecce461
diff --cc test/collection_vector_search_test.cppindex 390b9fa7,3ffe288e..2f55cd34--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@@ -1033,114 -1033,72 +1033,184 @@@ TEST_F(CollectionVectorTest, EmbedFromOASSERT_TRUE(add_op.ok());}+ TEST_F(CollectionVectorTest, SkipEmbeddingOpWhenValueExists) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ nlohmann::json model_config = R"({+ "model_name": "ts/e5-small"+ })"_json;++ // will be roughly 0.1110895648598671,-0.11710234731435776,-0.5319093465805054, ...++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ // document with explicit embedding vector+ nlohmann::json doc;+ doc["name"] = "FOO";++ std::vector<float> vec;+ for(size_t i = 0; i < 384; i++) {+ vec.push_back(0.345);+ }++ doc["embedding"] = vec;++ auto add_op = coll->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // get the vector back+ auto res = coll->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD).get();++ // let's check the first few vectors+ auto stored_vec = res["hits"][0]["document"]["embedding"];+ ASSERT_NEAR(0.345, stored_vec[0], 0.01);+ ASSERT_NEAR(0.345, stored_vec[1], 0.01);+ ASSERT_NEAR(0.345, stored_vec[2], 0.01);+ ASSERT_NEAR(0.345, stored_vec[3], 0.01);+ ASSERT_NEAR(0.345, stored_vec[4], 0.01);++ // what happens when vector contains invalid value, like string+ doc["embedding"] = "foo"; //{0.11, 0.11};+ add_op = coll->add(doc.dump());+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `embedding` contains an invalid embedding.", add_op.error());++ // when dims don't match+ doc["embedding"] = {0.11, 0.11};+ add_op = coll->add(doc.dump());+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `embedding` contains an invalid embedding.", add_op.error());++ // invalid array value+ doc["embedding"].clear();+ for(size_t i = 0; i < 384; i++) {+ doc["embedding"].push_back(0.01);+ }+ doc["embedding"][5] = "foo";+ add_op = coll->add(doc.dump());+ ASSERT_FALSE(add_op.ok());+ ASSERT_EQ("Field `embedding` contains invalid float values.", add_op.error());+ }+++TEST_F(CollectionVectorTest, SemanticSearchReturnOnlyVectorDistance) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;+++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "product_name": "moisturizer",+ "category": "beauty"+ })"_json.dump());++ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("moisturizer", {"embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ ASSERT_EQ(1, results["hits"].size());++ // Return only vector distance+ ASSERT_EQ(0, results["hits"][0].count("text_match_info"));+ ASSERT_EQ(0, results["hits"][0].count("hybrid_search_info"));+ ASSERT_EQ(1, results["hits"][0].count("vector_distance"));+}++TEST_F(CollectionVectorTest, KeywordSearchReturnOnlyTextMatchInfo) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;+++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ auto coll1 = collection_create_op.get();+ auto add_op = coll1->add(R"({+ "product_name": "moisturizer",+ "category": "beauty"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("moisturizer", {"product_name"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();+++ ASSERT_EQ(1, results["hits"].size());++ // Return only text match info+ ASSERT_EQ(0, results["hits"][0].count("vector_distance"));+ ASSERT_EQ(0, results["hits"][0].count("hybrid_search_info"));+ ASSERT_EQ(1, results["hits"][0].count("text_match_info"));+}++TEST_F(CollectionVectorTest, HybridSearchReturnAllInfo) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "category", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["product_name", "category"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ auto coll1 = collection_create_op.get();++ auto add_op = coll1->add(R"({+ "product_name": "moisturizer",+ "category": "beauty"+ })"_json.dump());+ ASSERT_TRUE(add_op.ok());+++ auto results = coll1->search("moisturizer", {"product_name", "embedding"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>()).get();++ ASSERT_EQ(1, results["hits"].size());++ // Return all info+ ASSERT_EQ(1, results["hits"][0].count("vector_distance"));+ ASSERT_EQ(1, results["hits"][0].count("text_match_info"));+ ASSERT_EQ(1, results["hits"][0].count("hybrid_search_info"));- }++}