Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor field-wise search configuration parsing.

Kishore Nallandcfd336b0619
diff --git a/include/collection.h b/include/collection.h
index 86474ef7..eb53ae5e 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -566,11 +566,9 @@ public:
Option<bool> alter(nlohmann::json& alter_payload);
- void
- process_search_field_weights(const std::vector<std::string>& raw_search_fields,
- std::vector<uint32_t>& query_by_weights,
- std::vector<search_field_t>& weighted_search_fields,
- std::vector<std::string>& reordered_search_fields) const;
+ void process_search_field_weights(const std::vector<search_field_t>& search_fields,
+ std::vector<uint32_t>& query_by_weights,
+ std::vector<search_field_t>& weighted_search_fields) const;
Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);
};
diff --git a/include/index.h b/include/index.h
index 51f3f05b..2937a914 100644
--- a/include/index.h
+++ b/include/index.h
@@ -72,12 +72,6 @@ struct tok_candidates {
std::vector<std::string> candidates;
};
-struct search_field_t {
- std::string name;
- size_t weight;
- size_t orig_index;
-};
-
struct query_tokens_t {
std::vector<token_t> q_include_tokens;
std::vector<std::vector<std::string>> q_exclude_tokens;
@@ -91,6 +85,17 @@ enum enable_t {
off
};
+struct search_field_t {
+ std::string name;
+ size_t weight;
+ size_t num_typos;
+ bool prefix;
+ enable_t infix;
+
+ search_field_t(const std::string& name, size_t weight, size_t num_typos, bool prefix, enable_t infix):
+ name(name), weight(weight), num_typos(num_typos), prefix(prefix), infix(infix) { }
+};
+
enum text_match_type_t {
max_score,
max_weight
diff --git a/src/collection.cpp b/src/collection.cpp
index 34f4ee2e..588acc1c 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1278,8 +1278,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
// validate search fields
- std::vector<std::string> processed_search_fields;
+ std::vector<search_field_t> processed_search_fields;
std::vector<uint32_t> query_by_weights;
+
size_t num_embed_fields = 0;
std::string query = raw_query;
@@ -1368,9 +1369,14 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
continue;
}
- processed_search_fields.push_back(expanded_search_field);
+ auto query_weight = !raw_query_by_weights.empty() ? raw_query_by_weights[i] : 0;
+ auto num_typo = i < num_typos.size() ? num_typos[i] : num_typos[0];
+ auto prefix = i < prefixes.size() ? prefixes[i] : prefixes[0];
+ auto infix = i < infixes.size() ? infixes[i] : infixes[0];
+
+ processed_search_fields.emplace_back(expanded_search_field, query_weight, num_typo, prefix, infix);
if(!raw_query_by_weights.empty()) {
- query_by_weights.push_back(raw_query_by_weights[i]);
+ query_by_weights.push_back(query_weight);
}
}
}
@@ -1391,7 +1397,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
- for(const std::string & field_name: processed_search_fields) {
+ for(const auto& processed_search_field: processed_search_fields) {
+ const auto& field_name = processed_search_field.name;
field search_field = search_schema.at(field_name);
if(!search_field.index) {
std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";
@@ -1440,12 +1447,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
// process weights for search fields
- std::vector<std::string> reordered_search_fields;
std::vector<search_field_t> weighted_search_fields;
- process_search_field_weights(processed_search_fields, query_by_weights, weighted_search_fields, reordered_search_fields);
-
- const std::vector<std::string>& search_fields = reordered_search_fields.empty() ? processed_search_fields
- : reordered_search_fields;
+ process_search_field_weights(processed_search_fields, query_by_weights, weighted_search_fields);
const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
filter_node_t* filter_tree_root = nullptr;
@@ -1535,7 +1538,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
size_t max_hits = DEFAULT_TOPSTER_SIZE;
// ensure that `max_hits` never exceeds number of documents in collection
- if(search_fields.size() <= 1 || query == "*") {
+ if(weighted_search_fields.size() <= 1 || query == "*") {
max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());
} else {
max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());
@@ -1651,7 +1654,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
std::vector<std::string> q_tokens; // used for auxillary highlighting
std::vector<std::string> q_include_tokens;
- if(search_fields.size() == 0) {
+ if(weighted_search_fields.size() == 0) {
// has to be a wildcard query
field_query_tokens.emplace_back(query_tokens_t{});
parse_search_query(query, q_include_tokens,
@@ -1689,7 +1692,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
}
- for(size_t i = 1; i < search_fields.size(); i++) {
+ for(size_t i = 1; i < weighted_search_fields.size(); i++) {
field_query_tokens.emplace_back(query_tokens_t{});
field_query_tokens[i] = field_query_tokens[0];
}
@@ -2304,35 +2307,36 @@ void Collection::copy_highlight_doc(std::vector<highlight_field_t>& hightlight_i
}
}
-void Collection::process_search_field_weights(const std::vector<std::string>& raw_search_fields,
+void Collection::process_search_field_weights(const std::vector<search_field_t>& search_fields,
std::vector<uint32_t>& query_by_weights,
- std::vector<search_field_t>& weighted_search_fields,
- std::vector<std::string>& reordered_search_fields) const {
+ std::vector<search_field_t>& weighted_search_fields) const {
const bool weights_given = !query_by_weights.empty();
// weights, if given, must be in desc order
bool weights_in_desc_order = true;
- bool weights_undex_max = true;
+ bool weights_under_max = true;
- for(size_t i=0; i < raw_search_fields.size(); i++) {
+ for(size_t i = 0; i < search_fields.size(); i++) {
if(!weights_given) {
size_t weight = std::max<int>(0, (int(Index::FIELD_MAX_WEIGHT) - i));
query_by_weights.push_back(weight);
- weighted_search_fields.push_back({raw_search_fields[i], weight, i});
+ auto wsearch_field = search_fields[i];
+ wsearch_field.weight = weight;
+ weighted_search_fields.push_back(wsearch_field);
} else {
// check if weights are already sorted
auto prev_weight = (i == 0) ? query_by_weights[0] : query_by_weights[i-1];
weights_in_desc_order = weights_in_desc_order && (query_by_weights[i] <= prev_weight);
- weights_undex_max = weights_undex_max && (query_by_weights[i] <= Index::FIELD_MAX_WEIGHT);
+ weights_under_max = weights_under_max && (query_by_weights[i] <= Index::FIELD_MAX_WEIGHT);
}
}
- if(weights_given && (!weights_in_desc_order || !weights_undex_max)) {
+ if(weights_given && (!weights_in_desc_order || !weights_under_max)) {
// ensure that search fields are sorted on their corresponding weight
std::vector<std::pair<size_t, size_t>> field_index_and_weights;
- for(size_t i=0; i < raw_search_fields.size(); i++) {
- field_index_and_weights.emplace_back(i, query_by_weights[i]);
+ for(size_t i=0; i < search_fields.size(); i++) {
+ field_index_and_weights.emplace_back(i, search_fields[i].weight);
}
std::sort(field_index_and_weights.begin(), field_index_and_weights.end(), [](const auto& a, const auto& b) {
@@ -2341,7 +2345,6 @@ void Collection::process_search_field_weights(const std::vector<std::string>& ra
for(size_t i = 0; i < field_index_and_weights.size(); i++) {
const auto& index_weight = field_index_and_weights[i];
- reordered_search_fields.push_back(raw_search_fields[index_weight.first]);
// we have to also normalize weights to 0 to Index::FIELD_MAX_WEIGHT range.
if(i == 0) {
@@ -2359,18 +2362,18 @@ void Collection::process_search_field_weights(const std::vector<std::string>& ra
}
}
- const auto& search_field = raw_search_fields[index_weight.first];
+ const auto& search_field = search_fields[index_weight.first];
const auto weight = query_by_weights[i];
const size_t orig_index = index_weight.first;
- weighted_search_fields.push_back({search_field, weight, orig_index});
+ auto wsearch_field = search_fields[orig_index];
+ wsearch_field.weight = weight;
+ weighted_search_fields.push_back(wsearch_field);
}
}
if(weighted_search_fields.empty()) {
- for(size_t i=0; i < raw_search_fields.size(); i++) {
- const auto& search_field = raw_search_fields[i];
- const auto weight = query_by_weights[i];
- weighted_search_fields.push_back({search_field, weight, i});
+ for(size_t i=0; i < search_fields.size(); i++) {
+ weighted_search_fields.push_back(search_fields[i]);
}
}
}
@@ -2443,8 +2446,7 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea
for(size_t i = 0; i < search_fields.size(); i++) {
const auto& field_name = search_fields[i].name;
- enable_t field_infix = (search_fields[i].orig_index < infixes.size()) ? infixes[search_fields[i].orig_index]
- : infixes[0];
+ enable_t field_infix = search_fields[i].infix;
if(field_infix != off) {
fields_infixed_set.insert(field_name);
}
diff --git a/src/index.cpp b/src/index.cpp
index 67ea75d7..aa55920b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3697,7 +3697,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
std::vector<size_t> query_field_ids(num_search_fields);
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
- query_field_ids[field_id] = the_fields[field_id].orig_index;
+ query_field_ids[field_id] = field_id;
}
std::vector<size_t> popular_field_ids; // fields containing the token most across documents
@@ -3718,14 +3718,14 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
// NOTE: when accessing other field ordered properties like prefixes or num_typos we have to index
// them by `the_field.orig_index` since the original fields could be reordered on their weights.
auto& the_field = the_fields[field_id];
- const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;
+ const bool field_prefix = the_fields[field_id].prefix;
const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
/*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"
<< token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/
- int64_t field_num_typos = (the_field.orig_index < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];
+ int64_t field_num_typos = the_fields[field_id].num_typos;
auto& locale = search_schema.at(the_field.name).locale;
if(locale != "" && (locale == "zh" || locale == "ko" || locale == "ja")) {
@@ -3777,10 +3777,10 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,
for(size_t field_id: query_field_ids) {
auto& the_field = the_fields[field_id];
- const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;
+ const bool field_prefix = the_fields[field_id].prefix;
const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;
const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;
- int64_t field_num_typos = (the_field.orig_index < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];
+ int64_t field_num_typos = the_fields[field_id].num_typos;
auto& locale = search_schema.at(the_field.name).locale;
if(locale != "" && locale != "en" && locale != "th" && !Tokenizer::is_cyrillic(locale)) {
@@ -4087,10 +4087,8 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
for(size_t i = 0; i < num_search_fields; i++) {
const std::string& field_name = the_fields[i].name;
- const uint32_t field_num_typos = (the_fields[i].orig_index < num_typos.size())
- ? num_typos[the_fields[i].orig_index] : num_typos[0];
- const bool field_prefix = (the_fields[i].orig_index < prefixes.size()) ? prefixes[the_fields[i].orig_index]
- : prefixes[0];
+ const uint32_t field_num_typos = the_fields[i].num_typos;
+ const bool field_prefix = the_fields[i].prefix;
if(token_num_typos > field_num_typos) {
// since the token can come from any field, we still have to respect per-field num_typos
@@ -4827,8 +4825,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
for(size_t field_id = 0; field_id < num_search_fields; field_id++) {
auto& field_name = the_fields[field_id].name;
- enable_t field_infix = (the_fields[field_id].orig_index < infixes.size())
- ? infixes[the_fields[field_id].orig_index] : infixes[0];
+ enable_t field_infix = the_fields[field_id].infix;
if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {
std::vector<uint32_t> infix_ids;
diff --git a/test/collection_infix_search_test.cpp b/test/collection_infix_search_test.cpp
index 6b2524f6..f1b11067 100644
--- a/test/collection_infix_search_test.cpp
+++ b/test/collection_infix_search_test.cpp
@@ -330,6 +330,48 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionInfixSearchTest, InfixOneOfManyFields) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "content", "type": "object"},
+ {"name": "data.title", "type": "string"},
+ {"name": "data.idClient", "type": "string"},
+ {"name": "data.jobNumber", "type": "string", "infix": true}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc = R"(
+ {
+ "data": {
+ "idFS": "xx",
+ "jobNumber": "XX_XX-EG00907",
+ "idClient": "862323",
+ "title": "my title"
+ },
+ "content": {
+ "task": "my task",
+ "description": "my description",
+ "status": "my status"
+ }
+ })"_json;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("EG00907",
+ {"data.title", "content", "data.idClient", "data.jobNumber"}, "", {}, {}, {0},
+ 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
+ 4, {off, off, off, always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+}
+
TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
field("points", field_types::INT32, false),};
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 86d0de41..b518d825 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1017,15 +1017,16 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
- std::vector<std::string> raw_search_fields = {"title", "brand", "type"};
+ std::vector<search_field_t> raw_search_fields = {
+ search_field_t("title", 110, 2, true, off),
+ search_field_t("brand", 25, 2, true, off),
+ search_field_t("type", 55, 2, true, off),
+ };
std::vector<uint32_t> query_by_weights = {110, 25, 55};
std::vector<search_field_t> weighted_search_fields;
- std::vector<std::string> reordered_search_fields;
- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,
- reordered_search_fields);
+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
- ASSERT_EQ(3, reordered_search_fields.size());
ASSERT_EQ(3, weighted_search_fields.size());
ASSERT_EQ("title", weighted_search_fields[0].name);
@@ -1038,11 +1039,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
// same weights
weighted_search_fields.clear();
- reordered_search_fields.clear();
query_by_weights = {15, 15, 15};
+ raw_search_fields = {
+ search_field_t{"title", 15, 2, true, off},
+ search_field_t{"brand", 15, 2, true, off},
+ search_field_t{"type", 15, 2, true, off},
+ };
- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,
- reordered_search_fields);
+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
ASSERT_EQ("title", weighted_search_fields[0].name);
ASSERT_EQ("brand", weighted_search_fields[1].name);
@@ -1054,11 +1058,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
// same weights large
weighted_search_fields.clear();
- reordered_search_fields.clear();
query_by_weights = {800, 800, 800};
+ raw_search_fields = {
+ search_field_t{"title", 800, 2, true, off},
+ search_field_t{"brand", 800, 2, true, off},
+ search_field_t{"type", 800, 2, true, off},
+ };
- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,
- reordered_search_fields);
+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
ASSERT_EQ("title", weighted_search_fields[0].name);
ASSERT_EQ("brand", weighted_search_fields[1].name);
@@ -1070,11 +1077,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
// weights desc ordered but exceed max weight
weighted_search_fields.clear();
- reordered_search_fields.clear();
query_by_weights = {603, 602, 601};
+ raw_search_fields = {
+ search_field_t{"title", 603, 2, true, off},
+ search_field_t{"brand", 602, 2, true, off},
+ search_field_t{"type", 601, 2, true, off},
+ };
- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,
- reordered_search_fields);
+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
ASSERT_EQ("title", weighted_search_fields[0].name);
ASSERT_EQ("brand", weighted_search_fields[1].name);
@@ -1087,16 +1097,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
// number of fields > 15 (must cap least important fields to weight 0)
raw_search_fields.clear();
weighted_search_fields.clear();
- reordered_search_fields.clear();
query_by_weights.clear();
for(size_t i = 0; i < 17; i++) {
- raw_search_fields.push_back("field" + std::to_string(17 - i));
+ raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 17 - i, 2, true, off});
query_by_weights.push_back(17 - i);
}
- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,
- reordered_search_fields);
+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
ASSERT_EQ("field3", weighted_search_fields[14].name);
ASSERT_EQ("field2", weighted_search_fields[15].name);
@@ -1109,15 +1117,13 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
// when weights are not given
raw_search_fields.clear();
weighted_search_fields.clear();
- reordered_search_fields.clear();
query_by_weights.clear();
for(size_t i = 0; i < 17; i++) {
- raw_search_fields.push_back("field" + std::to_string(17 - i));
+ raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 0, 2, true, off});
}
- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,
- reordered_search_fields);
+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);
ASSERT_EQ("field3", weighted_search_fields[14].name);
ASSERT_EQ("field2", weighted_search_fields[15].name);