Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix highlighting for fields with dot in their names.

Kishore Nallan81b67122965f
diff --git a/include/field.h b/include/field.h
index 981d4834..bd75334e 100644
--- a/include/field.h
+++ b/include/field.h
@@ -386,17 +386,20 @@ struct field {
return Option<bool>(true);
}
- static Option<bool> json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,
+ static Option<bool> json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,
+ std::vector<field>& the_fields,
string& fallback_field_type, size_t& num_auto_detect_fields);
- static Option<bool> json_fields_to_fields(nlohmann::json& fields_json,
+ static Option<bool> json_fields_to_fields(bool enable_nested_fields,
+ nlohmann::json& fields_json,
std::string& fallback_field_type,
std::vector<field>& the_fields) {
size_t num_auto_detect_fields = 0;
for(nlohmann::json & field_json: fields_json) {
- auto op = json_field_to_field(field_json, the_fields, fallback_field_type, num_auto_detect_fields);
+ auto op = json_field_to_field(enable_nested_fields,
+ field_json, the_fields, fallback_field_type, num_auto_detect_fields);
if(!op.ok()) {
return op;
}
diff --git a/src/collection.cpp b/src/collection.cpp
index ca4a577e..16b6c748 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2565,7 +2565,12 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
// in-place highlighting under the new highlight structure
std::vector<std::string> parts;
- StringUtils::split(search_field.name, parts, ".");
+ if(!enable_nested_fields || highlight_doc.contains(search_field.name)) {
+ parts = {search_field.name};
+ } else {
+ StringUtils::split(search_field.name, parts, ".");
+ }
+
nlohmann::json* hval = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;
nlohmann::json* fval = highlight_full_doc.contains(parts[0]) ? &highlight_full_doc[parts[0]] : nullptr;
@@ -3656,8 +3661,8 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
if(is_addition || is_reindex) {
// must validate fields
- auto parse_op = field::json_field_to_field(kv.value(), diff_fields, fallback_field_type,
- num_auto_detect_fields);
+ auto parse_op = field::json_field_to_field(enable_nested_fields, kv.value(), diff_fields,
+ fallback_field_type, num_auto_detect_fields);
if (!parse_op.ok()) {
return parse_op;
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 57972b20..024cfea8 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1088,7 +1088,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
std::string fallback_field_type;
std::vector<field> fields;
- auto parse_op = field::json_fields_to_fields(req_json["fields"], fallback_field_type, fields);
+ auto parse_op = field::json_fields_to_fields(req_json[ENABLE_NESTED_FIELDS].get<bool>(),
+ req_json["fields"], fallback_field_type, fields);
if(!parse_op.ok()) {
return Option<Collection*>(parse_op.code(), parse_op.error());
diff --git a/src/field.cpp b/src/field.cpp
index 4e43cc59..312e85ec 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -437,7 +437,8 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,
return Option<bool>(true);
}
-Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,
+Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,
+ std::vector<field>& the_fields,
string& fallback_field_type, size_t& num_auto_detect_fields) {
if(field_json["name"] == "id") {
@@ -614,7 +615,8 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<
bool is_obj = field_json[fields::type] == field_types::OBJECT || field_json[fields::type] == field_types::OBJECT_ARRAY;
bool is_regexp_name = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;
- if(is_obj || (!is_regexp_name && field_json[fields::name].get<std::string>().find('.') != std::string::npos)) {
+ if(is_obj || (!is_regexp_name && enable_nested_fields &&
+ field_json[fields::name].get<std::string>().find('.') != std::string::npos)) {
field_json[fields::nested] = true;
field_json[fields::nested_array] = field::VAL_UNKNOWN; // unknown, will be resolved during read
} else {
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 8b409e3f..ddce2f44 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -621,7 +621,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
std::string fallback_field_type;
std::vector<field> fields;
- auto parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ auto parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_TRUE(parse_op.ok());
ASSERT_EQ(1, fields.size());
@@ -639,7 +639,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
fields_json.emplace_back(string_star_field);
fields.clear();
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_TRUE(parse_op.ok());
ASSERT_EQ(true, fields[0].optional);
@@ -648,14 +648,14 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
// reject when you try to set optional to false or facet to true
fields_json[0][fields::optional] = false;
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_FALSE(parse_op.ok());
ASSERT_EQ("Field `.*` must be an optional field.", parse_op.error());
fields_json[0][fields::optional] = true;
fields_json[0][fields::facet] = true;
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_FALSE(parse_op.ok());
ASSERT_EQ("Field `.*` cannot be a facet field.", parse_op.error());
@@ -665,7 +665,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
// can have only one ".*" field
fields_json.emplace_back(all_field);
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_FALSE(parse_op.ok());
ASSERT_EQ("There can be only one field named `.*`.", parse_op.error());
@@ -676,7 +676,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
all_field[fields::type] = "auto";
fields_json.emplace_back(all_field);
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_TRUE(parse_op.ok());
ASSERT_EQ("auto", fields[0].type);
@@ -688,7 +688,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
all_field[fields::locale] = "ja";
fields_json.emplace_back(all_field);
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_TRUE(parse_op.ok());
ASSERT_EQ("ja", fields[0].locale);
@@ -700,7 +700,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {
all_field[fields::locale] = "ko";
fields_json.emplace_back(all_field);
- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);
+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);
ASSERT_TRUE(parse_op.ok());
ASSERT_EQ("ko", fields[0].locale);
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index f1c2945f..6cc38afe 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1619,3 +1619,47 @@ TEST_F(CollectionSpecificMoreTest, PhraseMatchMultipleFields) {
ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", res["hits"][1]["document"]["id"].get<std::string>());
}
+
+TEST_F(CollectionSpecificMoreTest, HighlightOnFieldNameWithDot) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "org.title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["org.title"] = "Infinity Inc.";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto res = coll1->search("infinity", {"org.title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(1, res["hits"][0]["highlights"].size());
+ ASSERT_EQ("<mark>Infinity</mark> Inc.", res["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ nlohmann::json snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;
+ ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());
+
+ // even if nested fields enabled, plain field names with dots should work fine
+
+ schema = R"({
+ "name": "coll2",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "org.title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll2 = collectionManager.create_collection(schema).get();
+ ASSERT_TRUE(coll2->add(doc.dump()).ok());
+
+ res = coll2->search("infinity", {"org.title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(0, res["hits"][0]["highlights"].size());
+ snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;
+ ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());
+}