Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Expose field weights used for scoring.
kishorenc66a44a5afce8
diff --git a/include/collection.h b/include/collection.hindex 5f46acc5..93ebb728 100644--- a/include/collection.h+++ b/include/collection.h@@ -358,7 +358,8 @@ public:const std::vector<std::string>& group_by_fields={},const size_t group_limit = 0,const std::string& highlight_start_tag="<mark>",- const std::string& highlight_end_tag="</mark>");+ const std::string& highlight_end_tag="</mark>",+ std::vector<size_t> query_by_weights={});Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/index.h b/include/index.hindex f3d0ad63..4ad2f1d6 100644--- a/include/index.h+++ b/include/index.h@@ -23,11 +23,16 @@ struct token_candidates {std::vector<art_leaf*> candidates;};+struct search_field_t {+ std::string name;+ size_t weight;+};+struct search_args {std::vector<std::string> q_include_tokens;std::vector<std::string> q_exclude_tokens;std::vector<std::vector<std::string>> q_synonyms;- std::vector<std::string> search_fields;+ std::vector<search_field_t> search_fields;std::vector<filter> filters;std::vector<facet> facets;std::map<size_t, std::map<size_t, uint32_t>> included_ids;@@ -60,16 +65,16 @@ struct search_args {search_args(std::vector<std::string> q_include_tokens,std::vector<std::string> q_exclude_tokens,std::vector<std::vector<std::string>> q_synonyms,- std::vector<std::string> search_fields, std::vector<filter> filters,+ std::vector<search_field_t> search_fields, std::vector<filter> filters,std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,size_t drop_tokens_threshold, size_t typo_tokens_threshold,const std::vector<std::string>& group_by_fields, size_t group_limit):q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens), q_synonyms(q_synonyms),- search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),- excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),- max_facet_values(max_facet_values), per_page(per_page),+ search_fields(search_fields), filters(filters), facets(facets),+ included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),+ facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),page(page), token_order(token_order), prefix(prefix),drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),group_by_fields(group_by_fields), group_limit(group_limit),@@ -227,7 +232,7 @@ public:const std::vector<std::string> & q_include_tokens,const std::vector<std::string> & q_exclude_tokens,const std::vector<std::vector<std::string>>& q_synonyms,- const std::vector<std::string> & search_fields,+ const std::vector<search_field_t> & search_fields,const std::vector<filter> & filters, std::vector<facet> & facets,facet_query_t & facet_query,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,diff --git a/src/collection.cpp b/src/collection.cppindex 63ea015c..630e8bfc 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -491,8 +491,8 @@ void Collection::populate_overrides(std::string query,}}-Option<nlohmann::json> Collection::search(const std::string & query, const std::vector<std::string> & search_fields,- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,+Option<nlohmann::json> Collection::search(const std::string & query, const std::vector<std::string>& search_fields,+ const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,const std::vector<sort_by> & sort_fields, const int num_typos,const size_t per_page, const size_t page,const token_ordering token_order, const bool prefix,@@ -510,12 +510,18 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const std::vector<std::string>& group_by_fields,const size_t group_limit,const std::string& highlight_start_tag,- const std::string& highlight_end_tag) {+ const std::string& highlight_end_tag,+ std::vector<size_t> query_by_weights) {if(query != "*" && search_fields.empty()) {return Option<nlohmann::json>(400, "No search fields specified for the query.");}+ if(!search_fields.empty() && !query_by_weights.empty() && search_fields.size() != query_by_weights.size()) {+ return Option<nlohmann::json>(400, "Number of weights in `query_by_weights` does not match "+ "number of `query_by` fields.");+ }+if(!group_by_fields.empty() && (group_limit == 0 || group_limit > GROUP_LIMIT_MAX)) {return Option<nlohmann::json>(400, "Value of `group_limit` must be between 1 and " +std::to_string(GROUP_LIMIT_MAX) + ".");@@ -579,6 +585,20 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::index_to_excluded_ids[index_id].push_back(seq_id);}+ // process weights for search fields+ std::vector<search_field_t> weighted_search_fields;++ if(query_by_weights.empty() && !search_fields.empty()) {+ for(int i=search_fields.size(); i > 0; i--) {+ query_by_weights.push_back(i);+ }+ }++ for(size_t i=0; i < search_fields.size(); i++) {+ const auto& search_field = search_fields[i];+ weighted_search_fields.push_back({search_field, query_by_weights[i]});+ }+std::vector<facet> facets;// validate search fields@@ -745,8 +765,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// send data to individual index threadssize_t index_id = 0;for(Index* index: indices) {- index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, search_fields, filters, facets,- index_to_included_ids[index_id], index_to_excluded_ids[index_id],+ index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,+ filters, facets, index_to_included_ids[index_id],+ index_to_excluded_ids[index_id],sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,per_page, page, token_order, prefix,drop_tokens_threshold, typo_tokens_threshold,diff --git a/src/core_api.cpp b/src/core_api.cppindex 97e647d6..a415096e 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -259,6 +259,7 @@ bool get_search(http_req & req, http_res & res) {const char *FILTER = "filter_by";const char *QUERY = "q";const char *QUERY_BY = "query_by";+ const char *QUERY_BY_WEIGHTS = "query_by_weights";const char *SORT_BY = "sort_by";const char *FACET_BY = "facet_by";@@ -372,6 +373,22 @@ bool get_search(http_req & req, http_res & res) {}}+ std::vector<std::string> query_by_weights_str;+ std::vector<size_t> query_by_weights;++ if(req.params.count(QUERY_BY_WEIGHTS) != 0) {+ StringUtils::split(req.params[QUERY_BY_WEIGHTS], query_by_weights_str, ",");+ for(const auto& weight_str: query_by_weights_str) {+ if(!StringUtils::is_uint32_t(weight_str)) {+ res.set_400("Parameter `" + std::string(QUERY_BY_WEIGHTS) ++ "` must be a comma separated string of unsigned integers.");+ return false;+ }++ query_by_weights.push_back(std::stoi(weight_str));+ }+ }+if(!StringUtils::is_uint32_t(req.params[DROP_TOKENS_THRESHOLD])) {res.set_400("Parameter `" + std::string(DROP_TOKENS_THRESHOLD) + "` must be an unsigned integer.");return false;@@ -505,7 +522,8 @@ bool get_search(http_req & req, http_res & res) {group_by_fields,static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),req.params[HIGHLIGHT_START_TAG],- req.params[HIGHLIGHT_END_TAG]+ req.params[HIGHLIGHT_END_TAG],+ query_by_weights);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex 47cd9f61..887af236 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1295,7 +1295,7 @@ void Index::search(Option<uint32_t> & outcome,const std::vector<std::string>& q_include_tokens,const std::vector<std::string>& q_exclude_tokens,const std::vector<std::vector<std::string>>& q_synonyms,- const std::vector<std::string>& search_fields,+ const std::vector<search_field_t>& search_fields,const std::vector<filter>& filters,std::vector<facet>& facets, facet_query_t& facet_query,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,@@ -1349,7 +1349,7 @@ void Index::search(Option<uint32_t> & outcome,// find documents that contain the excluded tokens to exclude them from results laterfor(size_t i = 0; i < num_search_fields; i++) {- const std::string & field_name = search_fields[i];+ const std::string & field_name = search_fields[i].name;for(const std::string& exclude_token: q_exclude_tokens) {art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),(const unsigned char *) exclude_token.c_str(),@@ -1372,7 +1372,7 @@ void Index::search(Option<uint32_t> & outcome,if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);- const std::string & field = search_fields[0];+ const std::string& field = search_fields[0].name;// if a filter is not specified, use the sorting index to generate the list of all document idsif(filters.empty()) {@@ -1438,7 +1438,7 @@ void Index::search(Option<uint32_t> & outcome,// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.empty() || filter_ids_length > 0) {const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);- const std::string & field = search_fields[i];+ const std::string& field = search_fields[i].name;std::vector<std::string> query_tokens = q_include_tokens;std::vector<std::string> search_tokens = q_include_tokens;@@ -1496,21 +1496,21 @@ void Index::search(Option<uint32_t> & outcome,const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);// produces weights that are powers of 2- size_t num_left_shifts = num_search_fields - i - 1;+ size_t weight = search_fields[i].weight;if(field_id == kvs[0]->field_id) {- aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] << num_left_shifts);+ aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] * weight);continue;}if(existing_field_kvs.count(field_id) != 0) {// for existing field, we will simply sum field-wise weighted scoresaggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]- << num_left_shifts);+ * weight);continue;}- const std::string & field = search_fields[i];+ const std::string& field = search_fields[i].name;// compute approximate match score for this field from actual query@@ -1549,7 +1549,7 @@ void Index::search(Option<uint32_t> & outcome,if(words_present != 0) {uint64_t match_score = Match::get_match_score(words_present, 0, 100, field_id);- aggregated_score += (match_score << num_left_shifts);+ aggregated_score += (match_score * weight);}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 94166cae..d811a153 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2696,6 +2696,108 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ // change weights to favor artist++ results = coll1->search("on a jetplane",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 4}).get();++ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionTest, FieldWeightsNotProper) {+ // when weights are not given properly+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ auto results_op = coll1->search("on a jetplane",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1});++ ASSERT_FALSE(results_op.ok());+ ASSERT_STREQ("Number of weights in `query_by_weights` does not match number "+ "of `query_by` fields.", results_op.error().c_str());++ results_op = coll1->search("on a jetplane",+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {2, 1});++ ASSERT_FALSE(results_op.ok());+ ASSERT_STREQ("Number of weights in `query_by_weights` does not match number "+ "of `query_by` fields.", results_op.error().c_str());++ // empty weights are fine (will be defaulted to)++ results_op = coll1->search("on a jetplane",+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {});++ ASSERT_TRUE(results_op.ok());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionTest, MultiFieldRelevance3) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Taylor Swift Karaoke: reputation", "Taylor Swift"},+ {"Style", "Taylor Swift"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("style taylor swift",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,+ true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {1, 1}).get();++ /*LOG(INFO) << results;++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());*/+collectionManager.drop_collection("coll1");}