Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Include embedding fields during export. (#1946)
Harpreet Sangareca71c3422fe
diff --git a/src/collection.cpp b/src/collection.cppindex 40d01b37..2708b4a5 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1556,9 +1556,9 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,bool field_found = false;for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {- bool exact_key_match = (kv.key().size() == field_name.size());- bool exact_primitive_match = exact_key_match && !kv.value().is_object();- bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().num_dim > 0;+ const bool exact_key_match = (kv.key().size() == field_name.size());+ const bool exact_primitive_match = exact_key_match && !kv.value().is_object();+ const bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().num_dim > 0;if(extract_only_string_fields && !kv.value().is_string() && !text_embedding) {if(exact_primitive_match && !is_wildcard) {@@ -1569,7 +1569,11 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,continue;}- if(!exact_key_match && text_embedding) {+ // Prefix matches should only be included if it is a wildcard field name or if the matched field is nested.+ // If we have the fields `title`, `title_ko`, and `title.foo`, and `title` is passed, it should only match `title`+ // and `title.foo` fields. `title*` should match all the fields.+ const bool is_nested_field = kv.value().nested;+ if(!exact_key_match && !is_wildcard && !is_nested_field) {continue;}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 8bb422c6..fd7260c5 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -3241,3 +3241,67 @@ TEST_F(CollectionSpecificMoreTest, EnableTyposForAlphaNumericalTokens) {ASSERT_EQ("(136)214", res["hits"][3]["document"]["title"].get<std::string>());ASSERT_EQ("13/14", res["hits"][4]["document"]["title"].get<std::string>());}++TEST_F(CollectionSpecificMoreTest, PopulateIncludeExcludeFields) {+ auto schema_json =+ R"({+ "name": "Products",+ "fields": [+ {"name": "product_id", "type": "string"},+ {"name": "product_name", "type": "string", "infix": true},+ {"name": "product_description", "type": "string"},+ {"name": "product_embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}},+ {"name": "rating", "type": "int32"}+ ]+ })"_json;+ std::vector<nlohmann::json> documents = {+ R"({+ "product_id": "product_a",+ "product_name": "shampoo",+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",+ "rating": "2"+ })"_json,+ R"({+ "product_id": "product_b",+ "product_name": "soap",+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",+ "rating": "4"+ })"_json+ };++ EmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ auto products_coll = collection_create_op.get();++ spp::sparse_hash_set<std::string> input_include_fields;+ spp::sparse_hash_set<std::string> input_exclude_fields;+ tsl::htrie_set<char> output_include_fields;+ tsl::htrie_set<char> output_exclude_fields;++ input_include_fields = {"product_*"};+ products_coll->populate_include_exclude_fields_lk(input_include_fields, input_exclude_fields, output_include_fields,+ output_exclude_fields);+ std::vector<std::string> expected = {"product_id", "product_name", "product_description"};++ for (const auto& item: expected) {+ ASSERT_EQ(1, output_include_fields.count(item));+ }++ input_include_fields = {"product_*"};+ products_coll->populate_include_exclude_fields_lk(input_include_fields, input_exclude_fields, output_include_fields,+ output_exclude_fields);+ expected = {"product_id", "product_name", "product_description", "product_embedding"};+ for (const auto& item: expected) {+ ASSERT_EQ(1, output_include_fields.count(item));+ }+}\ No newline at end of filediff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 3316a3ca..8229b482 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -4983,7 +4983,7 @@ TEST_F(CollectionVectorTest, TestDistanceThresholdWithIP) {std::map<std::string, std::string> req_params = {{"collection", "products"},{"q", "document"},- {"query_by", "*"},+ {"query_by", "name"},{"sort_by", "_text_match:desc,_vector_query(embedding:([0.11731103425347378, -0.6694758317235057, -0.6211945774857595, -0.27966758971688255, -0.4683744007950299],distance_threshold:1)):asc,rank_score:desc"},{"exclude_fields", "embedding"}};@@ -5005,7 +5005,7 @@ TEST_F(CollectionVectorTest, TestDistanceThresholdWithIP) {req_params = {{"collection", "products"},{"q", "document"},- {"query_by", "*"},+ {"query_by", "name"},{"sort_by", "_text_match:desc,_vector_query(embedding:([-100,-100,-100,-100,-100])):asc,rank_score:desc"},{"exclude_fields", "embedding"}};