Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor field-wise search configuration parsing.
Kishore Nallandcfd336b0619
diff --git a/include/collection.h b/include/collection.hindex 86474ef7..eb53ae5e 100644--- a/include/collection.h+++ b/include/collection.h@@ -566,11 +566,9 @@ public:Option<bool> alter(nlohmann::json& alter_payload);- void- process_search_field_weights(const std::vector<std::string>& raw_search_fields,- std::vector<uint32_t>& query_by_weights,- std::vector<search_field_t>& weighted_search_fields,- std::vector<std::string>& reordered_search_fields) const;+ void process_search_field_weights(const std::vector<search_field_t>& search_fields,+ std::vector<uint32_t>& query_by_weights,+ std::vector<search_field_t>& weighted_search_fields) const;Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);};diff --git a/include/index.h b/include/index.hindex 51f3f05b..2937a914 100644--- a/include/index.h+++ b/include/index.h@@ -72,12 +72,6 @@ struct tok_candidates {std::vector<std::string> candidates;};-struct search_field_t {- std::string name;- size_t weight;- size_t orig_index;-};-struct query_tokens_t {std::vector<token_t> q_include_tokens;std::vector<std::vector<std::string>> q_exclude_tokens;@@ -91,6 +85,17 @@ enum enable_t {off};+struct search_field_t {+ std::string name;+ size_t weight;+ size_t num_typos;+ bool prefix;+ enable_t infix;++ search_field_t(const std::string& name, size_t weight, size_t num_typos, bool prefix, enable_t infix):+ name(name), weight(weight), num_typos(num_typos), prefix(prefix), infix(infix) { }+};+enum text_match_type_t {max_score,max_weightdiff --git a/src/collection.cpp b/src/collection.cppindex 34f4ee2e..588acc1c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1278,8 +1278,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}// validate search fields- std::vector<std::string> processed_search_fields;+ std::vector<search_field_t> processed_search_fields;std::vector<uint32_t> query_by_weights;+size_t num_embed_fields = 0;std::string query = raw_query;@@ -1368,9 +1369,14 @@ Option<nlohmann::json> Collection::search(std::string raw_query,continue;}- processed_search_fields.push_back(expanded_search_field);+ auto query_weight = !raw_query_by_weights.empty() ? raw_query_by_weights[i] : 0;+ auto num_typo = i < num_typos.size() ? num_typos[i] : num_typos[0];+ auto prefix = i < prefixes.size() ? prefixes[i] : prefixes[0];+ auto infix = i < infixes.size() ? infixes[i] : infixes[0];++ processed_search_fields.emplace_back(expanded_search_field, query_weight, num_typo, prefix, infix);if(!raw_query_by_weights.empty()) {- query_by_weights.push_back(raw_query_by_weights[i]);+ query_by_weights.push_back(query_weight);}}}@@ -1391,7 +1397,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}- for(const std::string & field_name: processed_search_fields) {+ for(const auto& processed_search_field: processed_search_fields) {+ const auto& field_name = processed_search_field.name;field search_field = search_schema.at(field_name);if(!search_field.index) {std::string error = "Field `" + field_name + "` is marked as a non-indexed field in the schema.";@@ -1440,12 +1447,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}// process weights for search fields- std::vector<std::string> reordered_search_fields;std::vector<search_field_t> weighted_search_fields;- process_search_field_weights(processed_search_fields, query_by_weights, weighted_search_fields, reordered_search_fields);-- const std::vector<std::string>& search_fields = reordered_search_fields.empty() ? processed_search_fields- : reordered_search_fields;+ process_search_field_weights(processed_search_fields, query_by_weights, weighted_search_fields);const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";filter_node_t* filter_tree_root = nullptr;@@ -1535,7 +1538,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,size_t max_hits = DEFAULT_TOPSTER_SIZE;// ensure that `max_hits` never exceeds number of documents in collection- if(search_fields.size() <= 1 || query == "*") {+ if(weighted_search_fields.size() <= 1 || query == "*") {max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());} else {max_hits = std::min(std::max(fetch_size, max_hits), get_num_documents());@@ -1651,7 +1654,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::vector<std::string> q_tokens; // used for auxillary highlightingstd::vector<std::string> q_include_tokens;- if(search_fields.size() == 0) {+ if(weighted_search_fields.size() == 0) {// has to be a wildcard queryfield_query_tokens.emplace_back(query_tokens_t{});parse_search_query(query, q_include_tokens,@@ -1689,7 +1692,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}}- for(size_t i = 1; i < search_fields.size(); i++) {+ for(size_t i = 1; i < weighted_search_fields.size(); i++) {field_query_tokens.emplace_back(query_tokens_t{});field_query_tokens[i] = field_query_tokens[0];}@@ -2304,35 +2307,36 @@ void Collection::copy_highlight_doc(std::vector<highlight_field_t>& hightlight_i}}-void Collection::process_search_field_weights(const std::vector<std::string>& raw_search_fields,+void Collection::process_search_field_weights(const std::vector<search_field_t>& search_fields,std::vector<uint32_t>& query_by_weights,- std::vector<search_field_t>& weighted_search_fields,- std::vector<std::string>& reordered_search_fields) const {+ std::vector<search_field_t>& weighted_search_fields) const {const bool weights_given = !query_by_weights.empty();// weights, if given, must be in desc orderbool weights_in_desc_order = true;- bool weights_undex_max = true;+ bool weights_under_max = true;- for(size_t i=0; i < raw_search_fields.size(); i++) {+ for(size_t i = 0; i < search_fields.size(); i++) {if(!weights_given) {size_t weight = std::max<int>(0, (int(Index::FIELD_MAX_WEIGHT) - i));query_by_weights.push_back(weight);- weighted_search_fields.push_back({raw_search_fields[i], weight, i});+ auto wsearch_field = search_fields[i];+ wsearch_field.weight = weight;+ weighted_search_fields.push_back(wsearch_field);} else {// check if weights are already sortedauto prev_weight = (i == 0) ? query_by_weights[0] : query_by_weights[i-1];weights_in_desc_order = weights_in_desc_order && (query_by_weights[i] <= prev_weight);- weights_undex_max = weights_undex_max && (query_by_weights[i] <= Index::FIELD_MAX_WEIGHT);+ weights_under_max = weights_under_max && (query_by_weights[i] <= Index::FIELD_MAX_WEIGHT);}}- if(weights_given && (!weights_in_desc_order || !weights_undex_max)) {+ if(weights_given && (!weights_in_desc_order || !weights_under_max)) {// ensure that search fields are sorted on their corresponding weightstd::vector<std::pair<size_t, size_t>> field_index_and_weights;- for(size_t i=0; i < raw_search_fields.size(); i++) {- field_index_and_weights.emplace_back(i, query_by_weights[i]);+ for(size_t i=0; i < search_fields.size(); i++) {+ field_index_and_weights.emplace_back(i, search_fields[i].weight);}std::sort(field_index_and_weights.begin(), field_index_and_weights.end(), [](const auto& a, const auto& b) {@@ -2341,7 +2345,6 @@ void Collection::process_search_field_weights(const std::vector<std::string>& rafor(size_t i = 0; i < field_index_and_weights.size(); i++) {const auto& index_weight = field_index_and_weights[i];- reordered_search_fields.push_back(raw_search_fields[index_weight.first]);// we have to also normalize weights to 0 to Index::FIELD_MAX_WEIGHT range.if(i == 0) {@@ -2359,18 +2362,18 @@ void Collection::process_search_field_weights(const std::vector<std::string>& ra}}- const auto& search_field = raw_search_fields[index_weight.first];+ const auto& search_field = search_fields[index_weight.first];const auto weight = query_by_weights[i];const size_t orig_index = index_weight.first;- weighted_search_fields.push_back({search_field, weight, orig_index});+ auto wsearch_field = search_fields[orig_index];+ wsearch_field.weight = weight;+ weighted_search_fields.push_back(wsearch_field);}}if(weighted_search_fields.empty()) {- for(size_t i=0; i < raw_search_fields.size(); i++) {- const auto& search_field = raw_search_fields[i];- const auto weight = query_by_weights[i];- weighted_search_fields.push_back({search_field, weight, i});+ for(size_t i=0; i < search_fields.size(); i++) {+ weighted_search_fields.push_back(search_fields[i]);}}}@@ -2443,8 +2446,7 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& seafor(size_t i = 0; i < search_fields.size(); i++) {const auto& field_name = search_fields[i].name;- enable_t field_infix = (search_fields[i].orig_index < infixes.size()) ? infixes[search_fields[i].orig_index]- : infixes[0];+ enable_t field_infix = search_fields[i].infix;if(field_infix != off) {fields_infixed_set.insert(field_name);}diff --git a/src/index.cpp b/src/index.cppindex 67ea75d7..aa55920b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3697,7 +3697,7 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,std::vector<size_t> query_field_ids(num_search_fields);for(size_t field_id = 0; field_id < num_search_fields; field_id++) {- query_field_ids[field_id] = the_fields[field_id].orig_index;+ query_field_ids[field_id] = field_id;}std::vector<size_t> popular_field_ids; // fields containing the token most across documents@@ -3718,14 +3718,14 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,// NOTE: when accessing other field ordered properties like prefixes or num_typos we have to index// them by `the_field.orig_index` since the original fields could be reordered on their weights.auto& the_field = the_fields[field_id];- const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;+ const bool field_prefix = the_fields[field_id].prefix;const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;/*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"<< token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/- int64_t field_num_typos = (the_field.orig_index < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];+ int64_t field_num_typos = the_fields[field_id].num_typos;auto& locale = search_schema.at(the_field.name).locale;if(locale != "" && (locale == "zh" || locale == "ko" || locale == "ja")) {@@ -3777,10 +3777,10 @@ void Index::fuzzy_search_fields(const std::vector<search_field_t>& the_fields,for(size_t field_id: query_field_ids) {auto& the_field = the_fields[field_id];- const bool field_prefix = (the_field.orig_index < prefixes.size()) ? prefixes[the_field.orig_index] : prefixes[0];;+ const bool field_prefix = the_fields[field_id].prefix;const bool prefix_search = field_prefix && query_tokens[token_index].is_prefix_searched;const size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;- int64_t field_num_typos = (the_field.orig_index < num_typos.size()) ? num_typos[the_field.orig_index] : num_typos[0];+ int64_t field_num_typos = the_fields[field_id].num_typos;auto& locale = search_schema.at(the_field.name).locale;if(locale != "" && locale != "en" && locale != "th" && !Tokenizer::is_cyrillic(locale)) {@@ -4087,10 +4087,8 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,for(size_t i = 0; i < num_search_fields; i++) {const std::string& field_name = the_fields[i].name;- const uint32_t field_num_typos = (the_fields[i].orig_index < num_typos.size())- ? num_typos[the_fields[i].orig_index] : num_typos[0];- const bool field_prefix = (the_fields[i].orig_index < prefixes.size()) ? prefixes[the_fields[i].orig_index]- : prefixes[0];+ const uint32_t field_num_typos = the_fields[i].num_typos;+ const bool field_prefix = the_fields[i].prefix;if(token_num_typos > field_num_typos) {// since the token can come from any field, we still have to respect per-field num_typos@@ -4827,8 +4825,7 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<sefor(size_t field_id = 0; field_id < num_search_fields; field_id++) {auto& field_name = the_fields[field_id].name;- enable_t field_infix = (the_fields[field_id].orig_index < infixes.size())- ? infixes[the_fields[field_id].orig_index] : infixes[0];+ enable_t field_infix = the_fields[field_id].infix;if(field_infix == always || (field_infix == fallback && all_result_ids_len == 0)) {std::vector<uint32_t> infix_ids;diff --git a/test/collection_infix_search_test.cpp b/test/collection_infix_search_test.cppindex 6b2524f6..f1b11067 100644--- a/test/collection_infix_search_test.cpp+++ b/test/collection_infix_search_test.cpp@@ -330,6 +330,48 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionInfixSearchTest, InfixOneOfManyFields) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "content", "type": "object"},+ {"name": "data.title", "type": "string"},+ {"name": "data.idClient", "type": "string"},+ {"name": "data.jobNumber", "type": "string", "infix": true}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc = R"(+ {+ "data": {+ "idFS": "xx",+ "jobNumber": "XX_XX-EG00907",+ "idClient": "862323",+ "title": "my title"+ },+ "content": {+ "task": "my task",+ "description": "my description",+ "status": "my status"+ }+ })"_json;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("EG00907",+ {"data.title", "content", "data.idClient", "data.jobNumber"}, "", {}, {}, {0},+ 3, 1, FREQUENCY, {true}, 5,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,+ 4, {off, off, off, always}).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+}+TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),field("points", field_types::INT32, false),};diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 86d0de41..b518d825 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -1017,15 +1017,16 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();- std::vector<std::string> raw_search_fields = {"title", "brand", "type"};+ std::vector<search_field_t> raw_search_fields = {+ search_field_t("title", 110, 2, true, off),+ search_field_t("brand", 25, 2, true, off),+ search_field_t("type", 55, 2, true, off),+ };std::vector<uint32_t> query_by_weights = {110, 25, 55};std::vector<search_field_t> weighted_search_fields;- std::vector<std::string> reordered_search_fields;- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,- reordered_search_fields);+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);- ASSERT_EQ(3, reordered_search_fields.size());ASSERT_EQ(3, weighted_search_fields.size());ASSERT_EQ("title", weighted_search_fields[0].name);@@ -1038,11 +1039,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {// same weightsweighted_search_fields.clear();- reordered_search_fields.clear();query_by_weights = {15, 15, 15};+ raw_search_fields = {+ search_field_t{"title", 15, 2, true, off},+ search_field_t{"brand", 15, 2, true, off},+ search_field_t{"type", 15, 2, true, off},+ };- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,- reordered_search_fields);+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);ASSERT_EQ("title", weighted_search_fields[0].name);ASSERT_EQ("brand", weighted_search_fields[1].name);@@ -1054,11 +1058,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {// same weights largeweighted_search_fields.clear();- reordered_search_fields.clear();query_by_weights = {800, 800, 800};+ raw_search_fields = {+ search_field_t{"title", 800, 2, true, off},+ search_field_t{"brand", 800, 2, true, off},+ search_field_t{"type", 800, 2, true, off},+ };- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,- reordered_search_fields);+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);ASSERT_EQ("title", weighted_search_fields[0].name);ASSERT_EQ("brand", weighted_search_fields[1].name);@@ -1070,11 +1077,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {// weights desc ordered but exceed max weightweighted_search_fields.clear();- reordered_search_fields.clear();query_by_weights = {603, 602, 601};+ raw_search_fields = {+ search_field_t{"title", 603, 2, true, off},+ search_field_t{"brand", 602, 2, true, off},+ search_field_t{"type", 601, 2, true, off},+ };- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,- reordered_search_fields);+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);ASSERT_EQ("title", weighted_search_fields[0].name);ASSERT_EQ("brand", weighted_search_fields[1].name);@@ -1087,16 +1097,14 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {// number of fields > 15 (must cap least important fields to weight 0)raw_search_fields.clear();weighted_search_fields.clear();- reordered_search_fields.clear();query_by_weights.clear();for(size_t i = 0; i < 17; i++) {- raw_search_fields.push_back("field" + std::to_string(17 - i));+ raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 17 - i, 2, true, off});query_by_weights.push_back(17 - i);}- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,- reordered_search_fields);+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);ASSERT_EQ("field3", weighted_search_fields[14].name);ASSERT_EQ("field2", weighted_search_fields[15].name);@@ -1109,15 +1117,13 @@ TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {// when weights are not givenraw_search_fields.clear();weighted_search_fields.clear();- reordered_search_fields.clear();query_by_weights.clear();for(size_t i = 0; i < 17; i++) {- raw_search_fields.push_back("field" + std::to_string(17 - i));+ raw_search_fields.push_back(search_field_t{"field" + std::to_string(17 - i), 0, 2, true, off});}- coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields,- reordered_search_fields);+ coll1->process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields);ASSERT_EQ("field3", weighted_search_fields[14].name);ASSERT_EQ("field2", weighted_search_fields[15].name);