Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlighting for fields with dot in their names.
Kishore Nallan81b67122965f
diff --git a/include/field.h b/include/field.hindex 981d4834..bd75334e 100644--- a/include/field.h+++ b/include/field.h@@ -386,17 +386,20 @@ struct field {return Option<bool>(true);}- static Option<bool> json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,+ static Option<bool> json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,+ std::vector<field>& the_fields,string& fallback_field_type, size_t& num_auto_detect_fields);- static Option<bool> json_fields_to_fields(nlohmann::json& fields_json,+ static Option<bool> json_fields_to_fields(bool enable_nested_fields,+ nlohmann::json& fields_json,std::string& fallback_field_type,std::vector<field>& the_fields) {size_t num_auto_detect_fields = 0;for(nlohmann::json & field_json: fields_json) {- auto op = json_field_to_field(field_json, the_fields, fallback_field_type, num_auto_detect_fields);+ auto op = json_field_to_field(enable_nested_fields,+ field_json, the_fields, fallback_field_type, num_auto_detect_fields);if(!op.ok()) {return op;}diff --git a/src/collection.cpp b/src/collection.cppindex ca4a577e..16b6c748 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2565,7 +2565,12 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea// in-place highlighting under the new highlight structurestd::vector<std::string> parts;- StringUtils::split(search_field.name, parts, ".");+ if(!enable_nested_fields || highlight_doc.contains(search_field.name)) {+ parts = {search_field.name};+ } else {+ StringUtils::split(search_field.name, parts, ".");+ }+nlohmann::json* hval = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;nlohmann::json* fval = highlight_full_doc.contains(parts[0]) ? &highlight_full_doc[parts[0]] : nullptr;@@ -3656,8 +3661,8 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,if(is_addition || is_reindex) {// must validate fields- auto parse_op = field::json_field_to_field(kv.value(), diff_fields, fallback_field_type,- num_auto_detect_fields);+ auto parse_op = field::json_field_to_field(enable_nested_fields, kv.value(), diff_fields,+ fallback_field_type, num_auto_detect_fields);if (!parse_op.ok()) {return parse_op;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 57972b20..024cfea8 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1088,7 +1088,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsostd::string fallback_field_type;std::vector<field> fields;- auto parse_op = field::json_fields_to_fields(req_json["fields"], fallback_field_type, fields);+ auto parse_op = field::json_fields_to_fields(req_json[ENABLE_NESTED_FIELDS].get<bool>(),+ req_json["fields"], fallback_field_type, fields);if(!parse_op.ok()) {return Option<Collection*>(parse_op.code(), parse_op.error());diff --git a/src/field.cpp b/src/field.cppindex 4e43cc59..312e85ec 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -437,7 +437,8 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,return Option<bool>(true);}-Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<field>& the_fields,+Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::json& field_json,+ std::vector<field>& the_fields,string& fallback_field_type, size_t& num_auto_detect_fields) {if(field_json["name"] == "id") {@@ -614,7 +615,8 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<bool is_obj = field_json[fields::type] == field_types::OBJECT || field_json[fields::type] == field_types::OBJECT_ARRAY;bool is_regexp_name = field_json[fields::name].get<std::string>().find(".*") != std::string::npos;- if(is_obj || (!is_regexp_name && field_json[fields::name].get<std::string>().find('.') != std::string::npos)) {+ if(is_obj || (!is_regexp_name && enable_nested_fields &&+ field_json[fields::name].get<std::string>().find('.') != std::string::npos)) {field_json[fields::nested] = true;field_json[fields::nested_array] = field::VAL_UNKNOWN; // unknown, will be resolved during read} else {diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 8b409e3f..ddce2f44 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -621,7 +621,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {std::string fallback_field_type;std::vector<field> fields;- auto parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ auto parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_TRUE(parse_op.ok());ASSERT_EQ(1, fields.size());@@ -639,7 +639,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {fields_json.emplace_back(string_star_field);fields.clear();- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_TRUE(parse_op.ok());ASSERT_EQ(true, fields[0].optional);@@ -648,14 +648,14 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {// reject when you try to set optional to false or facet to truefields_json[0][fields::optional] = false;- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_FALSE(parse_op.ok());ASSERT_EQ("Field `.*` must be an optional field.", parse_op.error());fields_json[0][fields::optional] = true;fields_json[0][fields::facet] = true;- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_FALSE(parse_op.ok());ASSERT_EQ("Field `.*` cannot be a facet field.", parse_op.error());@@ -665,7 +665,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {// can have only one ".*" fieldfields_json.emplace_back(all_field);- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_FALSE(parse_op.ok());ASSERT_EQ("There can be only one field named `.*`.", parse_op.error());@@ -676,7 +676,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {all_field[fields::type] = "auto";fields_json.emplace_back(all_field);- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_TRUE(parse_op.ok());ASSERT_EQ("auto", fields[0].type);@@ -688,7 +688,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {all_field[fields::locale] = "ja";fields_json.emplace_back(all_field);- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_TRUE(parse_op.ok());ASSERT_EQ("ja", fields[0].locale);@@ -700,7 +700,7 @@ TEST_F(CollectionAllFieldsTest, JsonFieldsToFieldsConversion) {all_field[fields::locale] = "ko";fields_json.emplace_back(all_field);- parse_op = field::json_fields_to_fields(fields_json, fallback_field_type, fields);+ parse_op = field::json_fields_to_fields(false, fields_json, fallback_field_type, fields);ASSERT_TRUE(parse_op.ok());ASSERT_EQ("ko", fields[0].locale);}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex f1c2945f..6cc38afe 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -1619,3 +1619,47 @@ TEST_F(CollectionSpecificMoreTest, PhraseMatchMultipleFields) {ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", res["hits"][1]["document"]["id"].get<std::string>());}++TEST_F(CollectionSpecificMoreTest, HighlightOnFieldNameWithDot) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "org.title", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["org.title"] = "Infinity Inc.";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto res = coll1->search("infinity", {"org.title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(1, res["hits"][0]["highlights"].size());+ ASSERT_EQ("<mark>Infinity</mark> Inc.", res["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ nlohmann::json snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;+ ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());++ // even if nested fields enabled, plain field names with dots should work fine++ schema = R"({+ "name": "coll2",+ "enable_nested_fields": true,+ "fields": [+ {"name": "org.title", "type": "string"}+ ]+ })"_json;++ Collection* coll2 = collectionManager.create_collection(schema).get();+ ASSERT_TRUE(coll2->add(doc.dump()).ok());++ res = coll2->search("infinity", {"org.title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(0, res["hits"][0]["highlights"].size());+ snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;+ ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());+}