Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Include embedding fields during export. (#1946)

Harpreet Sangareca71c3422fe
diff --git a/src/collection.cpp b/src/collection.cpp
index 40d01b37..2708b4a5 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1556,9 +1556,9 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
bool field_found = false;
for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {
- bool exact_key_match = (kv.key().size() == field_name.size());
- bool exact_primitive_match = exact_key_match && !kv.value().is_object();
- bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().num_dim > 0;
+ const bool exact_key_match = (kv.key().size() == field_name.size());
+ const bool exact_primitive_match = exact_key_match && !kv.value().is_object();
+ const bool text_embedding = kv.value().type == field_types::FLOAT_ARRAY && kv.value().num_dim > 0;
if(extract_only_string_fields && !kv.value().is_string() && !text_embedding) {
if(exact_primitive_match && !is_wildcard) {
@@ -1569,7 +1569,11 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
continue;
}
- if(!exact_key_match && text_embedding) {
+ // Prefix matches should only be included if it is a wildcard field name or if the matched field is nested.
+ // If we have the fields `title`, `title_ko`, and `title.foo`, and `title` is passed, it should only match `title`
+ // and `title.foo` fields. `title*` should match all the fields.
+ const bool is_nested_field = kv.value().nested;
+ if(!exact_key_match && !is_wildcard && !is_nested_field) {
continue;
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 8bb422c6..fd7260c5 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -3241,3 +3241,67 @@ TEST_F(CollectionSpecificMoreTest, EnableTyposForAlphaNumericalTokens) {
ASSERT_EQ("(136)214", res["hits"][3]["document"]["title"].get<std::string>());
ASSERT_EQ("13/14", res["hits"][4]["document"]["title"].get<std::string>());
}
+
+TEST_F(CollectionSpecificMoreTest, PopulateIncludeExcludeFields) {
+ auto schema_json =
+ R"({
+ "name": "Products",
+ "fields": [
+ {"name": "product_id", "type": "string"},
+ {"name": "product_name", "type": "string", "infix": true},
+ {"name": "product_description", "type": "string"},
+ {"name": "product_embedding", "type":"float[]", "embed":{"from": ["product_description"], "model_config": {"model_name": "ts/e5-small"}}},
+ {"name": "rating", "type": "int32"}
+ ]
+ })"_json;
+ std::vector<nlohmann::json> documents = {
+ R"({
+ "product_id": "product_a",
+ "product_name": "shampoo",
+ "product_description": "Our new moisturizing shampoo is perfect for those with dry or damaged hair.",
+ "rating": "2"
+ })"_json,
+ R"({
+ "product_id": "product_b",
+ "product_name": "soap",
+ "product_description": "Introducing our all-natural, organic soap bar made with essential oils and botanical ingredients.",
+ "rating": "4"
+ })"_json
+ };
+
+ EmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ auto products_coll = collection_create_op.get();
+
+ spp::sparse_hash_set<std::string> input_include_fields;
+ spp::sparse_hash_set<std::string> input_exclude_fields;
+ tsl::htrie_set<char> output_include_fields;
+ tsl::htrie_set<char> output_exclude_fields;
+
+ input_include_fields = {"product_*"};
+ products_coll->populate_include_exclude_fields_lk(input_include_fields, input_exclude_fields, output_include_fields,
+ output_exclude_fields);
+ std::vector<std::string> expected = {"product_id", "product_name", "product_description"};
+
+ for (const auto& item: expected) {
+ ASSERT_EQ(1, output_include_fields.count(item));
+ }
+
+ input_include_fields = {"product_*"};
+ products_coll->populate_include_exclude_fields_lk(input_include_fields, input_exclude_fields, output_include_fields,
+ output_exclude_fields);
+ expected = {"product_id", "product_name", "product_description", "product_embedding"};
+ for (const auto& item: expected) {
+ ASSERT_EQ(1, output_include_fields.count(item));
+ }
+}
\ No newline at end of file
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 3316a3ca..8229b482 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -4983,7 +4983,7 @@ TEST_F(CollectionVectorTest, TestDistanceThresholdWithIP) {
std::map<std::string, std::string> req_params = {
{"collection", "products"},
{"q", "document"},
- {"query_by", "*"},
+ {"query_by", "name"},
{"sort_by", "_text_match:desc,_vector_query(embedding:([0.11731103425347378, -0.6694758317235057, -0.6211945774857595, -0.27966758971688255, -0.4683744007950299],distance_threshold:1)):asc,rank_score:desc"},
{"exclude_fields", "embedding"}
};
@@ -5005,7 +5005,7 @@ TEST_F(CollectionVectorTest, TestDistanceThresholdWithIP) {
req_params = {
{"collection", "products"},
{"q", "document"},
- {"query_by", "*"},
+ {"query_by", "name"},
{"sort_by", "_text_match:desc,_vector_query(embedding:([-100,-100,-100,-100,-100])):asc,rank_score:desc"},
{"exclude_fields", "embedding"}
};