Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix ordered weighted field index properties.

Kishore Nallan6bb2e6e6f885
diff --git a/include/collection.h b/include/collection.h
index c6ab24af..7bd40485 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -400,7 +400,7 @@ public:
const uint8_t* index_symbols,
size_t snippet_start_offset) ;
- void process_highlight_fields(const std::vector<std::string>& search_fields,
+ void process_highlight_fields(const std::vector<search_field_t>& search_fields,
const spp::sparse_hash_set<std::string>& exclude_fields,
const spp::sparse_hash_set<std::string>& include_fields,
const string& highlight_fields,
diff --git a/include/index.h b/include/index.h
index f75f83b2..e91e7efb 100644
--- a/include/index.h
+++ b/include/index.h
@@ -70,6 +70,7 @@ struct tok_candidates {
struct search_field_t {
std::string name;
size_t weight;
+ size_t orig_index;
};
struct query_tokens_t {
diff --git a/src/collection.cpp b/src/collection.cpp
index 7bf0561b..b8c04310 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1092,7 +1092,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
}
} else {
field_query_tokens.emplace_back(query_tokens_t{});
- const std::string & field_locale = search_schema.at(search_fields[0]).locale;
+ const std::string & field_locale = search_schema.at(weighted_search_fields[0].name).locale;
parse_search_query(query, q_include_tokens,
field_query_tokens[0].q_exclude_tokens,
field_query_tokens[0].q_phrases,
@@ -1266,7 +1266,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
std::vector<highlight_field_t> highlight_items;
if(query != "*") {
- process_highlight_fields(search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,
+ process_highlight_fields(weighted_search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,
infixes, q_tokens, search_params->qtoken_set, highlight_items);
}
@@ -1594,7 +1594,7 @@ void Collection::process_search_field_weights(const std::vector<std::string>& ra
if(!weights_given) {
size_t weight = std::max<int>(0, (int(Index::FIELD_MAX_WEIGHT) - i));
query_by_weights.push_back(weight);
- weighted_search_fields.push_back({raw_search_fields[i], weight});
+ weighted_search_fields.push_back({raw_search_fields[i], weight, i});
} else {
// check if weights are already sorted
auto prev_weight = (i == 0) ? query_by_weights[0] : query_by_weights[i-1];
@@ -1634,17 +1634,19 @@ void Collection::process_search_field_weights(const std::vector<std::string>& ra
query_by_weights[i] = bounded_weight;
}
}
+
+ const auto& search_field = raw_search_fields[index_weight.first];
+ const auto weight = query_by_weights[i];
+ const size_t orig_index = index_weight.first;
+ weighted_search_fields.push_back({search_field, weight, orig_index});
}
}
if(weighted_search_fields.empty()) {
- const std::vector<std::string>& search_fields = reordered_search_fields.empty() ? raw_search_fields
- : reordered_search_fields;
-
- for(size_t i=0; i < search_fields.size(); i++) {
- const auto& search_field = search_fields[i];
+ for(size_t i=0; i < raw_search_fields.size(); i++) {
+ const auto& search_field = raw_search_fields[i];
const auto weight = query_by_weights[i];
- weighted_search_fields.push_back({search_field, weight});
+ weighted_search_fields.push_back({search_field, weight, i});
}
}
}
@@ -1663,7 +1665,7 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_s
info["fields_matched"] = ((match_score << 61) >> (61));
}
-void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,
+void Collection::process_highlight_fields(const std::vector<search_field_t>& search_fields,
const spp::sparse_hash_set<std::string>& include_fields,
const spp::sparse_hash_set<std::string>& exclude_fields,
const string& highlight_fields,
@@ -1685,9 +1687,9 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
spp::sparse_hash_set<std::string> fields_infixed_set;
for(size_t i = 0; i < search_fields.size(); i++) {
- const auto& field_name = search_fields[i];
+ const auto& field_name = search_fields[i].name;
- enable_t field_infix = (i < infixes.size()) ? infixes[i] : infixes[0];
+ enable_t field_infix = (i < infixes.size()) ? infixes[search_fields[i].orig_index] : infixes[0];
if(field_infix != off) {
fields_infixed_set.insert(field_name);
}
@@ -1695,7 +1697,7 @@ void Collection::process_highlight_fields(const std::vector<std::string>& search
if(highlight_fields.empty()) {
for(size_t i = 0; i < search_fields.size(); i++) {
- const auto& field_name = search_fields[i];
+ const auto& field_name = search_fields[i].name;
if(exclude_fields.count(field_name) != 0) {
// should not pick excluded field for highlighting (only for implicit highlighting)
continue;
diff --git a/src/index.cpp b/src/index.cpp
index 24850e07..5f9d8eaf 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2814,15 +2814,17 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
//auto begin = std::chrono::high_resolution_clock::now();
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
+ // NOTE: when accessing other field ordered properties like prefixes or num_typos we have to index
+ // them by `the_field.orig_index` since the original fields could be reordered on their weights.
auto& the_field = the_fields[field_id];
- const bool field_prefix = (field_id < prefixes.size()) ? prefixes[field_id] : prefixes[0];;
+ const bool field_prefix = (field_id < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;
const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
/*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"
<< token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/
- int64_t field_num_typos = (field_id < num_typos.size()) ? num_typos[field_id] : num_typos[0];
+ int64_t field_num_typos = (field_id < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];
auto& locale = search_schema.at(the_field.name).locale;
if(locale != "" && locale != "en" && !Tokenizer::is_cyrillic(locale)) {
@@ -3618,7 +3620,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
- enable_t field_infix = (field_id < infixes.size()) ? infixes[field_id] : infixes[0];
+ enable_t field_infix = (field_id < infixes.size()) ? infixes[the_fields[field_id].orig_index] : infixes[0];
if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {
std::vector<uint32_t> infix_ids;
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index eaa91a2d..32be11ae 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -991,3 +991,44 @@ TEST_F(CollectionSpecificMoreTest, SearchingForMinusCharacter) {
ASSERT_EQ(1, results["hits"].size());
}
+
+TEST_F(CollectionSpecificMoreTest, UnorderedWeightingOfFields) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "brand", "type": "string"},
+ {"name": "sku", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "42f05db9-373a-4372-9bd0-ff4b5aaba28d";
+ doc["brand"] = "brand";
+ doc["sku"] = "rgx761";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ // with num_typos
+
+ auto res_op = coll1->search("rg0761", {"title","brand","sku"}, "", {}, {}, {2,2,0}, 10, 1,
+ FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {10, 7, 10});
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(0, res_op.get()["hits"].size());
+
+ // with prefix
+ res_op = coll1->search("rgx", {"title","brand","sku"}, "", {}, {}, {2,2,0}, 10, 1,
+ FREQUENCY, {true, true, false},
+ 0, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {10, 7, 10});
+
+ ASSERT_TRUE(res_op.ok());
+ ASSERT_EQ(0, res_op.get()["hits"].size());
+}