Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix ordered weighted field index properties.
Kishore Nallan6bb2e6e6f885
diff --git a/include/collection.h b/include/collection.hindex c6ab24af..7bd40485 100644--- a/include/collection.h+++ b/include/collection.h@@ -400,7 +400,7 @@ public:const uint8_t* index_symbols,size_t snippet_start_offset) ;- void process_highlight_fields(const std::vector<std::string>& search_fields,+ void process_highlight_fields(const std::vector<search_field_t>& search_fields,const spp::sparse_hash_set<std::string>& exclude_fields,const spp::sparse_hash_set<std::string>& include_fields,const string& highlight_fields,diff --git a/include/index.h b/include/index.hindex f75f83b2..e91e7efb 100644--- a/include/index.h+++ b/include/index.h@@ -70,6 +70,7 @@ struct tok_candidates {struct search_field_t {std::string name;size_t weight;+ size_t orig_index;};struct query_tokens_t {diff --git a/src/collection.cpp b/src/collection.cppindex 7bf0561b..b8c04310 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1092,7 +1092,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}} else {field_query_tokens.emplace_back(query_tokens_t{});- const std::string & field_locale = search_schema.at(search_fields[0]).locale;+ const std::string & field_locale = search_schema.at(weighted_search_fields[0].name).locale;parse_search_query(query, q_include_tokens,field_query_tokens[0].q_exclude_tokens,field_query_tokens[0].q_phrases,@@ -1266,7 +1266,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,std::vector<highlight_field_t> highlight_items;if(query != "*") {- process_highlight_fields(search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,+ process_highlight_fields(weighted_search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,infixes, q_tokens, search_params->qtoken_set, highlight_items);}@@ -1594,7 +1594,7 @@ void Collection::process_search_field_weights(const std::vector<std::string>& raif(!weights_given) {size_t weight = std::max<int>(0, (int(Index::FIELD_MAX_WEIGHT) - i));query_by_weights.push_back(weight);- weighted_search_fields.push_back({raw_search_fields[i], weight});+ weighted_search_fields.push_back({raw_search_fields[i], weight, i});} else {// check if weights are already sortedauto prev_weight = (i == 0) ? query_by_weights[0] : query_by_weights[i-1];@@ -1634,17 +1634,19 @@ void Collection::process_search_field_weights(const std::vector<std::string>& raquery_by_weights[i] = bounded_weight;}}++ const auto& search_field = raw_search_fields[index_weight.first];+ const auto weight = query_by_weights[i];+ const size_t orig_index = index_weight.first;+ weighted_search_fields.push_back({search_field, weight, orig_index});}}if(weighted_search_fields.empty()) {- const std::vector<std::string>& search_fields = reordered_search_fields.empty() ? raw_search_fields- : reordered_search_fields;-- for(size_t i=0; i < search_fields.size(); i++) {- const auto& search_field = search_fields[i];+ for(size_t i=0; i < raw_search_fields.size(); i++) {+ const auto& search_field = raw_search_fields[i];const auto weight = query_by_weights[i];- weighted_search_fields.push_back({search_field, weight});+ weighted_search_fields.push_back({search_field, weight, i});}}}@@ -1663,7 +1665,7 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_sinfo["fields_matched"] = ((match_score << 61) >> (61));}-void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,+void Collection::process_highlight_fields(const std::vector<search_field_t>& search_fields,const spp::sparse_hash_set<std::string>& include_fields,const spp::sparse_hash_set<std::string>& exclude_fields,const string& highlight_fields,@@ -1685,9 +1687,9 @@ void Collection::process_highlight_fields(const std::vector<std::string>& searchspp::sparse_hash_set<std::string> fields_infixed_set;for(size_t i = 0; i < search_fields.size(); i++) {- const auto& field_name = search_fields[i];+ const auto& field_name = search_fields[i].name;- enable_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];+ enable_t field_infix = (i < infixes.size()) ? infixes[search_fields[i].orig_index] : infixes[0];if(field_infix != off) {fields_infixed_set.insert(field_name);}@@ -1695,7 +1697,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& searchif(highlight_fields.empty()) {for(size_t i = 0; i < search_fields.size(); i++) {- const auto& field_name = search_fields[i];+ const auto& field_name = search_fields[i].name;if(exclude_fields.count(field_name) != 0) {// should not pick excluded field for highlighting (only for implicit highlighting)continue;diff --git a/src/index.cpp b/src/index.cppindex 24850e07..5f9d8eaf 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2814,15 +2814,17 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,//auto begin = std::chrono::high_resolution_clock::now();for(size_t field_id = 0; field_id < num_search_fields; field_id++) {+ // NOTE: when accessing other field ordered properties like prefixes or num_typos we have to index+ // them by `the_field.orig_index` since the original fields could be reordered on their weights.auto& the_field = the_fields[field_id];- const bool field_prefix = (field_id < prefixes.size()) ? prefixes[field_id] : prefixes[0];;+ const bool field_prefix = (field_id < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;/*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"<< token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/- int64_t field_num_typos = (field_id < num_typos.size()) ? num_typos[field_id] : num_typos[0];+ int64_t field_num_typos = (field_id < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];auto& locale = search_schema.at(the_field.name).locale;if(locale != "" && locale != "en" && !Tokenizer::is_cyrillic(locale)) {@@ -3618,7 +3620,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<sefor(size_t field_id = 0; field_id < num_search_fields; field_id++) {auto& field_name = the_fields[field_id].name;- enable_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];+ enable_t field_infix = (field_id < infixes.size()) ? infixes[the_fields[field_id].orig_index] : infixes[0];if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {std::vector<uint32_t> infix_ids;diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex eaa91a2d..32be11ae 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -991,3 +991,44 @@ TEST_F(CollectionSpecificMoreTest, SearchingForMinusCharacter) {ASSERT_EQ(1, results["hits"].size());}++TEST_F(CollectionSpecificMoreTest, UnorderedWeightingOfFields) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "brand", "type": "string"},+ {"name": "sku", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "42f05db9-373a-4372-9bd0-ff4b5aaba28d";+ doc["brand"] = "brand";+ doc["sku"] = "rgx761";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ // with num_typos++ auto res_op = coll1->search("rg0761", {"title","brand","sku"}, "", {}, {}, {2,2,0}, 10, 1,+ FREQUENCY, {true},+ 0, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {10, 7, 10});++ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(0, res_op.get()["hits"].size());++ // with prefix+ res_op = coll1->search("rgx", {"title","brand","sku"}, "", {}, {}, {2,2,0}, 10, 1,+ FREQUENCY, {true, true, false},+ 0, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {10, 7, 10});++ ASSERT_TRUE(res_op.ok());+ ASSERT_EQ(0, res_op.get()["hits"].size());+}