Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix edge case with field names containing dots.

Kishore Nallan643ba73ec78f
diff --git a/include/collection.h b/include/collection.h
index 7f1a4b83..3f785124 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -398,7 +398,7 @@ public:
size_t group_limit = 3,
const std::string& highlight_start_tag="<mark>",
const std::string& highlight_end_tag="</mark>",
- std::vector<uint32_t> query_by_weights={},
+ std::vector<uint32_t> raw_query_by_weights={},
size_t limit_hits=UINT32_MAX,
bool prioritize_exact_match=true,
bool pre_segmented_query=false,
diff --git a/src/collection.cpp b/src/collection.cpp
index 68122e9f..93fba46b 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -793,23 +793,21 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
bool field_found = false;
for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {
+ bool exact_key_match = (kv.key().size() == field_name.size());
+ bool exact_primitive_match = exact_key_match && !kv.value().is_object();
+
if(extract_only_string_fields && !kv.value().is_string()) {
- if(kv.value().nested && !enable_nested_fields) {
- continue;
+ if(exact_primitive_match) {
+ // upstream needs to be returned an error
+ return Option<bool>(400, "Field `" + field_name + "` should be a string or a string array.");
}
- if(kv.key().size() == field_name.size() && !kv.value().is_object()) {
- // exact key, must be rejected because of type mismatch
- std::string error = "Field `" + field_name + "` should be a string or a string array.";;
- return Option<bool>(400, error);
- }
continue;
}
- bool exact_non_obj_match = (kv.key().size() == field_name.size() && !kv.value().is_object());
-
// field_name prefix must be followed by a "." to indicate an object search
- if(exact_non_obj_match || (kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.')) {
+ if (exact_primitive_match || (enable_nested_fields && kv.key().size() > field_name.size() &&
+ kv.key()[field_name.size()] == '.')) {
processed_search_fields.push_back(kv.key());
field_found = true;
}
@@ -844,7 +842,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
size_t group_limit,
const std::string& highlight_start_tag,
const std::string& highlight_end_tag,
- std::vector<uint32_t> query_by_weights,
+ std::vector<uint32_t> raw_query_by_weights,
size_t limit_hits,
bool prioritize_exact_match,
bool pre_segmented_query,
@@ -875,7 +873,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
return Option<nlohmann::json>(400, "No search fields specified for the query.");
}
- if(!raw_search_fields.empty() && !query_by_weights.empty() && raw_search_fields.size() != query_by_weights.size()) {
+ if(!raw_search_fields.empty() && !raw_query_by_weights.empty() &&
+ raw_search_fields.size() != raw_query_by_weights.size()) {
return Option<nlohmann::json>(400, "Number of weights in `query_by_weights` does not match "
"number of `query_by` fields.");
}
@@ -887,7 +886,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
if(!raw_search_fields.empty() && raw_search_fields.size() != num_typos.size()) {
if(num_typos.size() != 1) {
- return Option<nlohmann::json>(400, "Number of weights in `num_typos` does not match "
+ return Option<nlohmann::json>(400, "Number of values in `num_typos` does not match "
"number of `query_by` fields.");
}
}
@@ -933,18 +932,33 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
// validate search fields
std::vector<std::string> processed_search_fields;
+ std::vector<uint32_t> query_by_weights;
- for(const std::string& field_name: raw_search_fields) {
+ for(size_t i = 0; i < raw_search_fields.size(); i++) {
+ const std::string& field_name = raw_search_fields[i];
if(field_name == "id") {
// `id` field needs to be handled separately, we will not handle for now
std::string error = "Cannot use `id` as a query by field.";
return Option<nlohmann::json>(400, error);
}
- auto field_op = extract_field_name(field_name, search_schema, processed_search_fields, true, enable_nested_fields);
+ std::vector<std::string> expanded_search_fields;
+ auto field_op = extract_field_name(field_name, search_schema, expanded_search_fields, true, enable_nested_fields);
if(!field_op.ok()) {
return Option<nlohmann::json>(field_op.code(), field_op.error());
}
+
+ for(const auto& expanded_search_field: expanded_search_fields) {
+ processed_search_fields.push_back(expanded_search_field);
+ if(!raw_query_by_weights.empty()) {
+ query_by_weights.push_back(raw_query_by_weights[i]);
+ }
+ }
+ }
+
+ if(!query_by_weights.empty() && processed_search_fields.size() != query_by_weights.size()) {
+ std::string error = "Error, query_by_weights.size != query_by.size.";
+ return Option<nlohmann::json>(400, error);
}
for(const std::string & field_name: processed_search_fields) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index e0ed4181..360f3282 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -496,6 +496,8 @@ bool post_multi_search(const std::shared_ptr<http_req>& req, const std::shared_p
return false;
}
+ LOG(INFO) << "REQ: " << req_json.dump(-1);
+
for(size_t i = 0; i < searches.size(); i++) {
auto& search_params = searches[i];
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 3895f9ad..58950b71 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -1299,7 +1299,34 @@ TEST_F(CollectionNestedFieldsTest, ExplicitDotSeparatedFieldsShouldHavePrecenden
results = coll2->search("*", {}, "company.ids: 1", {}, sort_fields, {0}, 10, 1,
token_ordering::FREQUENCY, {true}).get();
ASSERT_EQ(0, results["found"].get<size_t>());
+}
+TEST_F(CollectionNestedFieldsTest, NestedFieldWithExplicitWeight) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": ".*", "type": "auto"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc1 = R"({
+ "id": "0",
+ "company": {"num_employees": 2000, "founded": 1976},
+ "studies": [{"name": "College 1", "location": "USA"}]
+ })"_json;
+
+ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
+
+ auto results = coll1->search("college", {"studies"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "category", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
}
TEST_F(CollectionNestedFieldsTest, GroupByOnNestedFieldsWithWildcardSchema) {
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 90dd6679..3450a6fb 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1477,4 +1477,34 @@ TEST_F(CollectionSpecificMoreTest, ValidateQueryById) {
auto res_op = coll1->search("doc-1", {"id"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0);
ASSERT_FALSE(res_op.ok());
ASSERT_EQ("Cannot use `id` as a query by field.", res_op.error());
+}
+
+TEST_F(CollectionSpecificMoreTest, NonNestedFieldNameWithDot) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "category", "type": "string"},
+ {"name": "category.lvl0", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["category"] = "Shoes";
+ doc["category.lvl0"] = "Shoes";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["category"] = "Mens";
+ doc["category.lvl0"] = "Shoes";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto res = coll1->search("shoes", {"category"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "category", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1}).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
}
\ No newline at end of file