Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix edge case with field names containing dots.
Kishore Nallan643ba73ec78f
diff --git a/include/collection.h b/include/collection.hindex 7f1a4b83..3f785124 100644--- a/include/collection.h+++ b/include/collection.h@@ -398,7 +398,7 @@ public:size_t group_limit = 3,const std::string& highlight_start_tag="<mark>",const std::string& highlight_end_tag="</mark>",- std::vector<uint32_t> query_by_weights={},+ std::vector<uint32_t> raw_query_by_weights={},size_t limit_hits=UINT32_MAX,bool prioritize_exact_match=true,bool pre_segmented_query=false,diff --git a/src/collection.cpp b/src/collection.cppindex 68122e9f..93fba46b 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -793,23 +793,21 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,bool field_found = false;for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {+ bool exact_key_match = (kv.key().size() == field_name.size());+ bool exact_primitive_match = exact_key_match && !kv.value().is_object();+if(extract_only_string_fields && !kv.value().is_string()) {- if(kv.value().nested && !enable_nested_fields) {- continue;+ if(exact_primitive_match) {+ // upstream needs to be returned an error+ return Option<bool>(400, "Field `" + field_name + "` should be a string or a string array.");}- if(kv.key().size() == field_name.size() && !kv.value().is_object()) {- // exact key, must be rejected because of type mismatch- std::string error = "Field `" + field_name + "` should be a string or a string array.";;- return Option<bool>(400, error);- }continue;}- bool exact_non_obj_match = (kv.key().size() == field_name.size() && !kv.value().is_object());-// field_name prefix must be followed by a "." to indicate an object search- if(exact_non_obj_match || (kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.')) {+ if (exact_primitive_match || (enable_nested_fields && kv.key().size() > field_name.size() &&+ kv.key()[field_name.size()] == '.')) {processed_search_fields.push_back(kv.key());field_found = true;}@@ -844,7 +842,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,size_t group_limit,const std::string& highlight_start_tag,const std::string& highlight_end_tag,- std::vector<uint32_t> query_by_weights,+ std::vector<uint32_t> raw_query_by_weights,size_t limit_hits,bool prioritize_exact_match,bool pre_segmented_query,@@ -875,7 +873,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,return Option<nlohmann::json>(400, "No search fields specified for the query.");}- if(!raw_search_fields.empty() && !query_by_weights.empty() && raw_search_fields.size() != query_by_weights.size()) {+ if(!raw_search_fields.empty() && !raw_query_by_weights.empty() &&+ raw_search_fields.size() != raw_query_by_weights.size()) {return Option<nlohmann::json>(400, "Number of weights in `query_by_weights` does not match ""number of `query_by` fields.");}@@ -887,7 +886,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,if(!raw_search_fields.empty() && raw_search_fields.size() != num_typos.size()) {if(num_typos.size() != 1) {- return Option<nlohmann::json>(400, "Number of weights in `num_typos` does not match "+ return Option<nlohmann::json>(400, "Number of values in `num_typos` does not match ""number of `query_by` fields.");}}@@ -933,18 +932,33 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,// validate search fieldsstd::vector<std::string> processed_search_fields;+ std::vector<uint32_t> query_by_weights;- for(const std::string& field_name: raw_search_fields) {+ for(size_t i = 0; i < raw_search_fields.size(); i++) {+ const std::string& field_name = raw_search_fields[i];if(field_name == "id") {// `id` field needs to be handled separately, we will not handle for nowstd::string error = "Cannot use `id` as a query by field.";return Option<nlohmann::json>(400, error);}- auto field_op = extract_field_name(field_name, search_schema, processed_search_fields, true, enable_nested_fields);+ std::vector<std::string> expanded_search_fields;+ auto field_op = extract_field_name(field_name, search_schema, expanded_search_fields, true, enable_nested_fields);if(!field_op.ok()) {return Option<nlohmann::json>(field_op.code(), field_op.error());}++ for(const auto& expanded_search_field: expanded_search_fields) {+ processed_search_fields.push_back(expanded_search_field);+ if(!raw_query_by_weights.empty()) {+ query_by_weights.push_back(raw_query_by_weights[i]);+ }+ }+ }++ if(!query_by_weights.empty() && processed_search_fields.size() != query_by_weights.size()) {+ std::string error = "Error, query_by_weights.size != query_by.size.";+ return Option<nlohmann::json>(400, error);}for(const std::string & field_name: processed_search_fields) {diff --git a/src/core_api.cpp b/src/core_api.cppindex e0ed4181..360f3282 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -496,6 +496,8 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_preturn false;}+ LOG(INFO) << "REQ: " << req_json.dump(-1);+for(size_t i = 0; i < searches.size(); i++) {auto& search_params = searches[i];diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 3895f9ad..58950b71 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -1299,7 +1299,34 @@ TEST_F(CollectionNestedFieldsTest, ExplicitDotSeparatedFieldsShouldHavePrecendenresults = coll2->search("*", {}, "company.ids: 1", {}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}).get();ASSERT_EQ(0, results["found"].get<size_t>());+}+TEST_F(CollectionNestedFieldsTest, NestedFieldWithExplicitWeight) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": ".*", "type": "auto"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "id": "0",+ "company": {"num_employees": 2000, "founded": 1976},+ "studies": [{"name": "College 1", "location": "USA"}]+ })"_json;++ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());++ auto results = coll1->search("college", {"studies"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "category", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {1}).get();+ ASSERT_EQ(1, results["found"].get<size_t>());}TEST_F(CollectionNestedFieldsTest, GroupByOnNestedFieldsWithWildcardSchema) {diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 90dd6679..3450a6fb 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -1477,4 +1477,34 @@ TEST_F(CollectionSpecificMoreTest, ValidateQueryById) {auto res_op = coll1->search("doc-1", {"id"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0);ASSERT_FALSE(res_op.ok());ASSERT_EQ("Cannot use `id` as a query by field.", res_op.error());+}++TEST_F(CollectionSpecificMoreTest, NonNestedFieldNameWithDot) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "category", "type": "string"},+ {"name": "category.lvl0", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["category"] = "Shoes";+ doc["category.lvl0"] = "Shoes";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["category"] = "Mens";+ doc["category.lvl0"] = "Shoes";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto res = coll1->search("shoes", {"category"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "category", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {1}).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());}\ No newline at end of file