Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Expose field weights used for scoring.

kishorenc66a44a5afce8
diff --git a/include/collection.h b/include/collection.h
index 5f46acc5..93ebb728 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -358,7 +358,8 @@ public:
const std::vector<std::string>& group_by_fields={},
const size_t group_limit = 0,
const std::string& highlight_start_tag="<mark>",
- const std::string& highlight_end_tag="</mark>");
+ const std::string& highlight_end_tag="</mark>",
+ std::vector<size_t> query_by_weights={});
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/include/index.h b/include/index.h
index f3d0ad63..4ad2f1d6 100644
--- a/include/index.h
+++ b/include/index.h
@@ -23,11 +23,16 @@ struct token_candidates {
std::vector<art_leaf*> candidates;
};
+struct search_field_t {
+ std::string name;
+ size_t weight;
+};
+
struct search_args {
std::vector<std::string> q_include_tokens;
std::vector<std::string> q_exclude_tokens;
std::vector<std::vector<std::string>> q_synonyms;
- std::vector<std::string> search_fields;
+ std::vector<search_field_t> search_fields;
std::vector<filter> filters;
std::vector<facet> facets;
std::map<size_t, std::map<size_t, uint32_t>> included_ids;
@@ -60,16 +65,16 @@ struct search_args {
search_args(std::vector<std::string> q_include_tokens,
std::vector<std::string> q_exclude_tokens,
std::vector<std::vector<std::string>> q_synonyms,
- std::vector<std::string> search_fields, std::vector<filter> filters,
+ std::vector<search_field_t> search_fields, std::vector<filter> filters,
std::vector<facet> facets, std::map<size_t, std::map<size_t, uint32_t>> included_ids, std::vector<uint32_t> excluded_ids,
std::vector<sort_by> sort_fields_std, facet_query_t facet_query, int num_typos, size_t max_facet_values,
size_t max_hits, size_t per_page, size_t page, token_ordering token_order, bool prefix,
size_t drop_tokens_threshold, size_t typo_tokens_threshold,
const std::vector<std::string>& group_by_fields, size_t group_limit):
q_include_tokens(q_include_tokens), q_exclude_tokens(q_exclude_tokens), q_synonyms(q_synonyms),
- search_fields(search_fields), filters(filters), facets(facets), included_ids(included_ids),
- excluded_ids(excluded_ids), sort_fields_std(sort_fields_std), facet_query(facet_query), num_typos(num_typos),
- max_facet_values(max_facet_values), per_page(per_page),
+ search_fields(search_fields), filters(filters), facets(facets),
+ included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
+ facet_query(facet_query), num_typos(num_typos), max_facet_values(max_facet_values), per_page(per_page),
page(page), token_order(token_order), prefix(prefix),
drop_tokens_threshold(drop_tokens_threshold), typo_tokens_threshold(typo_tokens_threshold),
group_by_fields(group_by_fields), group_limit(group_limit),
@@ -227,7 +232,7 @@ public:
const std::vector<std::string> & q_include_tokens,
const std::vector<std::string> & q_exclude_tokens,
const std::vector<std::vector<std::string>>& q_synonyms,
- const std::vector<std::string> & search_fields,
+ const std::vector<search_field_t> & search_fields,
const std::vector<filter> & filters, std::vector<facet> & facets,
facet_query_t & facet_query,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
diff --git a/src/collection.cpp b/src/collection.cpp
index 63ea015c..630e8bfc 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -491,8 +491,8 @@ void Collection::populate_overrides(std::string query,
}
}
-Option<nlohmann::json> Collection::search(const std::string & query, const std::vector<std::string> & search_fields,
- const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,
+Option<nlohmann::json> Collection::search(const std::string & query, const std::vector<std::string>& search_fields,
+ const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,
const std::vector<sort_by> & sort_fields, const int num_typos,
const size_t per_page, const size_t page,
const token_ordering token_order, const bool prefix,
@@ -510,12 +510,18 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
const std::vector<std::string>& group_by_fields,
const size_t group_limit,
const std::string& highlight_start_tag,
- const std::string& highlight_end_tag) {
+ const std::string& highlight_end_tag,
+ std::vector<size_t> query_by_weights) {
if(query != "*" && search_fields.empty()) {
return Option<nlohmann::json>(400, "No search fields specified for the query.");
}
+ if(!search_fields.empty() && !query_by_weights.empty() && search_fields.size() != query_by_weights.size()) {
+ return Option<nlohmann::json>(400, "Number of weights in `query_by_weights` does not match "
+ "number of `query_by` fields.");
+ }
+
if(!group_by_fields.empty() && (group_limit == 0 || group_limit > GROUP_LIMIT_MAX)) {
return Option<nlohmann::json>(400, "Value of `group_limit` must be between 1 and " +
std::to_string(GROUP_LIMIT_MAX) + ".");
@@ -579,6 +585,20 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
index_to_excluded_ids[index_id].push_back(seq_id);
}
+ // process weights for search fields
+ std::vector<search_field_t> weighted_search_fields;
+
+ if(query_by_weights.empty() && !search_fields.empty()) {
+ for(int i=search_fields.size(); i > 0; i--) {
+ query_by_weights.push_back(i);
+ }
+ }
+
+ for(size_t i=0; i < search_fields.size(); i++) {
+ const auto& search_field = search_fields[i];
+ weighted_search_fields.push_back({search_field, query_by_weights[i]});
+ }
+
std::vector<facet> facets;
// validate search fields
@@ -745,8 +765,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
// send data to individual index threads
size_t index_id = 0;
for(Index* index: indices) {
- index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, search_fields, filters, facets,
- index_to_included_ids[index_id], index_to_excluded_ids[index_id],
+ index->search_params = new search_args(q_include_tokens, q_exclude_tokens, q_synonyms, weighted_search_fields,
+ filters, facets, index_to_included_ids[index_id],
+ index_to_excluded_ids[index_id],
sort_fields_std, facet_query, num_typos, max_facet_values, max_hits,
per_page, page, token_order, prefix,
drop_tokens_threshold, typo_tokens_threshold,
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 97e647d6..a415096e 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -259,6 +259,7 @@ bool get_search(http_req & req, http_res & res) {
const char *FILTER = "filter_by";
const char *QUERY = "q";
const char *QUERY_BY = "query_by";
+ const char *QUERY_BY_WEIGHTS = "query_by_weights";
const char *SORT_BY = "sort_by";
const char *FACET_BY = "facet_by";
@@ -372,6 +373,22 @@ bool get_search(http_req & req, http_res & res) {
}
}
+ std::vector<std::string> query_by_weights_str;
+ std::vector<size_t> query_by_weights;
+
+ if(req.params.count(QUERY_BY_WEIGHTS) != 0) {
+ StringUtils::split(req.params[QUERY_BY_WEIGHTS], query_by_weights_str, ",");
+ for(const auto& weight_str: query_by_weights_str) {
+ if(!StringUtils::is_uint32_t(weight_str)) {
+ res.set_400("Parameter `" + std::string(QUERY_BY_WEIGHTS) +
+ "` must be a comma separated string of unsigned integers.");
+ return false;
+ }
+
+ query_by_weights.push_back(std::stoi(weight_str));
+ }
+ }
+
if(!StringUtils::is_uint32_t(req.params[DROP_TOKENS_THRESHOLD])) {
res.set_400("Parameter `" + std::string(DROP_TOKENS_THRESHOLD) + "` must be an unsigned integer.");
return false;
@@ -505,7 +522,8 @@ bool get_search(http_req & req, http_res & res) {
group_by_fields,
static_cast<size_t>(std::stol(req.params[GROUP_LIMIT])),
req.params[HIGHLIGHT_START_TAG],
- req.params[HIGHLIGHT_END_TAG]
+ req.params[HIGHLIGHT_END_TAG],
+ query_by_weights
);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
diff --git a/src/index.cpp b/src/index.cpp
index 47cd9f61..887af236 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1295,7 +1295,7 @@ void Index::search(Option<uint32_t> & outcome,
const std::vector<std::string>& q_include_tokens,
const std::vector<std::string>& q_exclude_tokens,
const std::vector<std::vector<std::string>>& q_synonyms,
- const std::vector<std::string>& search_fields,
+ const std::vector<search_field_t>& search_fields,
const std::vector<filter>& filters,
std::vector<facet>& facets, facet_query_t& facet_query,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
@@ -1349,7 +1349,7 @@ void Index::search(Option<uint32_t> & outcome,
// find documents that contain the excluded tokens to exclude them from results later
for(size_t i = 0; i < num_search_fields; i++) {
- const std::string & field_name = search_fields[i];
+ const std::string & field_name = search_fields[i].name;
for(const std::string& exclude_token: q_exclude_tokens) {
art_leaf* leaf = (art_leaf *) art_search(search_index.at(field_name),
(const unsigned char *) exclude_token.c_str(),
@@ -1372,7 +1372,7 @@ void Index::search(Option<uint32_t> & outcome,
if(!q_include_tokens.empty() && q_include_tokens[0] == "*") {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
- const std::string & field = search_fields[0];
+ const std::string& field = search_fields[0].name;
// if a filter is not specified, use the sorting index to generate the list of all document ids
if(filters.empty()) {
@@ -1438,7 +1438,7 @@ void Index::search(Option<uint32_t> & outcome,
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.empty() || filter_ids_length > 0) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
- const std::string & field = search_fields[i];
+ const std::string& field = search_fields[i].name;
std::vector<std::string> query_tokens = q_include_tokens;
std::vector<std::string> search_tokens = q_include_tokens;
@@ -1496,21 +1496,21 @@ void Index::search(Option<uint32_t> & outcome,
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
// produces weights that are powers of 2
- size_t num_left_shifts = num_search_fields - i - 1;
+ size_t weight = search_fields[i].weight;
if(field_id == kvs[0]->field_id) {
- aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] << num_left_shifts);
+ aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] * weight);
continue;
}
if(existing_field_kvs.count(field_id) != 0) {
// for existing field, we will simply sum field-wise weighted scores
aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]
- << num_left_shifts);
+ * weight);
continue;
}
- const std::string & field = search_fields[i];
+ const std::string& field = search_fields[i].name;
// compute approximate match score for this field from actual query
@@ -1549,7 +1549,7 @@ void Index::search(Option<uint32_t> & outcome,
if(words_present != 0) {
uint64_t match_score = Match::get_match_score(words_present, 0, 100, field_id);
- aggregated_score += (match_score << num_left_shifts);
+ aggregated_score += (match_score * weight);
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 94166cae..d811a153 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2696,6 +2696,108 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ // change weights to favor artist
+
+ results = coll1->search("on a jetplane",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 4}).get();
+
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionTest, FieldWeightsNotProper) {
+ // when weights are not given properly
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ auto results_op = coll1->search("on a jetplane",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1});
+
+ ASSERT_FALSE(results_op.ok());
+ ASSERT_STREQ("Number of weights in `query_by_weights` does not match number "
+ "of `query_by` fields.", results_op.error().c_str());
+
+ results_op = coll1->search("on a jetplane",
+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {2, 1});
+
+ ASSERT_FALSE(results_op.ok());
+ ASSERT_STREQ("Number of weights in `query_by_weights` does not match number "
+ "of `query_by` fields.", results_op.error().c_str());
+
+ // empty weights are fine (will be defaulted to)
+
+ results_op = coll1->search("on a jetplane",
+ {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {});
+
+ ASSERT_TRUE(results_op.ok());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionTest, MultiFieldRelevance3) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Taylor Swift Karaoke: reputation", "Taylor Swift"},
+ {"Style", "Taylor Swift"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("style taylor swift",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 1}).get();
+
+ /*LOG(INFO) << results;
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());*/
+
collectionManager.drop_collection("coll1");
}