Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Ensure that weights can fully control cross-field matching.

Kishore Nallan89a509513ab4
diff --git a/include/index.h b/include/index.h
index 65564625..37bb45e0 100644
--- a/include/index.h
+++ b/include/index.h
@@ -35,6 +35,7 @@ struct token_candidates {
struct search_field_t {
std::string name;
+ size_t priority;
size_t weight;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index cb4ee5d9..0624cb44 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -600,21 +600,22 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
// process weights for search fields
std::vector<search_field_t> weighted_search_fields;
+ size_t max_weight = 100;
if(query_by_weights.empty()) {
+ max_weight = search_fields.size();
for(size_t i=1; i <= search_fields.size(); i++) {
- query_by_weights.push_back(i);
+ query_by_weights.push_back((max_weight - i) + 1);
}
} else {
- auto max_weight = *std::max_element(query_by_weights.begin(), query_by_weights.end());
- for(size_t i=0; i < query_by_weights.size(); i++) {
- query_by_weights[i] = (max_weight - query_by_weights[i]) + 1;
- }
+ max_weight = *std::max_element(query_by_weights.begin(), query_by_weights.end());
}
for(size_t i=0; i < search_fields.size(); i++) {
const auto& search_field = search_fields[i];
- weighted_search_fields.push_back({search_field, query_by_weights[i]});
+ const auto priority = (max_weight - query_by_weights[i]) + 1;
+ const auto weight = query_by_weights[i] + 1;
+ weighted_search_fields.push_back({search_field, priority, weight});
}
std::vector<facet> facets;
diff --git a/src/index.cpp b/src/index.cpp
index d1350620..6f6f4b7d 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1707,17 +1707,24 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
uint64_t total_typos = 0, total_distances = 0, min_typos = 1000;
- uint64_t verbatim_match_fields = 0; // query matching field verbatim
- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens
- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)
+ uint64_t verbatim_match_fields = 0; // query matching field verbatim
+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens
+ uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field
+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)
//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
for(size_t i = 0; i < num_search_fields; i++) {
const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
- size_t weight = search_fields[i].weight;
+ const size_t priority = search_fields[i].priority;
+ const size_t weight = search_fields[i].weight;
- //LOG(INFO) << "--- field index: " << i << ", weight: " << weight;
+ //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;
+
+ // using `5` here because typo + prefix combo score range is: 0 - 5
+ // 0 1 2
+ // 0,1 2,3 4,5
+ int64_t MAX_SUM_TYPOS = 5 * field_query_tokens[i].q_include_tokens.size();
if(existing_field_kvs.count(field_id) != 0) {
// for existing field, we will simply sum field-wise weighted scores
@@ -1728,8 +1735,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint64_t tokens_found = ((match_score >> 24) & 0xFF);
uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);
- total_typos += (field_typos + 1) * weight;
- total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * weight;
+ total_typos += (field_typos + 1) * priority;
+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;
verbatim_match_fields += (((match_score & 0xFF)) + 1);
uint64_t unique_tokens_found =
@@ -1739,6 +1746,11 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
exact_match_fields++;
}
+ auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);
+ if(weighted_tokens_match > max_weighted_tokens_match) {
+ max_weighted_tokens_match = weighted_tokens_match;
+ }
+
if(field_typos < min_typos) {
min_typos = field_typos;
}
@@ -1746,9 +1758,9 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
total_token_matches += tokens_found;
/*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))
- << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * weight
+ << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * priority
<< ", total dist: " << (((match_score & 0xFF)))
- << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * weight;*/
+ << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * priority;*/
continue;
}
@@ -1792,14 +1804,20 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint64_t tokens_found = ((match_score >> 24) & 0xFF);
uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);
- total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * weight;
- total_typos += (field_typos + 1) * weight;
+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;
+ total_typos += (field_typos + 1) * priority;
if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {
exact_match_fields++;
verbatim_match_fields++; // this is only an approximate
}
+ auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);
+
+ if(weighted_tokens_match > max_weighted_tokens_match) {
+ max_weighted_tokens_match = weighted_tokens_match;
+ }
+
if(field_typos < min_typos) {
min_typos = field_typos;
}
@@ -1814,9 +1832,11 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
total_typos = std::min<uint64_t>(255, total_typos);
total_distances = std::min<uint64_t>(100, total_distances);
+ max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);
uint64_t aggregated_score = (
- (exact_match_fields << 48) | // number of fields that contain *all tokens* in the query
+ //(exact_match_fields << 48) | // number of fields that contain *all tokens* in the query
+ (max_weighted_tokens_match << 48) | // weighted max number of tokens matched in a field
(uniq_tokens_found << 40) | // number of unique tokens found across fields including typos
((255 - min_typos) << 32) | // minimum typo cost across all fields
(total_token_matches << 24) | // total matches across fields including typos
@@ -1829,6 +1849,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
/*LOG(INFO) << "seq id: " << seq_id
<< ", exact_match_fields: " << exact_match_fields
+ << ", max_weighted_tokens_match: " << max_weighted_tokens_match
<< ", uniq_tokens_found: " << uniq_tokens_found
<< ", min typo score: " << (255 - min_typos)
<< ", total_token_matches: " << total_token_matches
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index 147dad3c..011e7527 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -21,7 +21,7 @@ std::string get_query(StringUtils & string_utils, std::string & text) {
for(uint32_t i=0; i<tokens.size(); i++) {
auto token = tokens[i];
- string_utils.unicode_normalize(token);
+ //string_utils.unicode_normalize(token);
normalized_tokens.push_back(token);
}
@@ -87,7 +87,7 @@ void benchmark_hn_titles(char* file_path) {
auto begin = std::chrono::high_resolution_clock::now();
for(size_t i = 0; i < queries.size(); i++) {
- auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("points", "DESC")}, 2, 10, 1, MAX_SCORE, true);
+ auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("points", "DESC")}, {2}, 10, 1, MAX_SCORE, {true});
if(results_op.ok() != true) {
exit(2);
}
@@ -152,8 +152,8 @@ void benchmark_reactjs_pages(char* file_path) {
auto begin = std::chrono::high_resolution_clock::now();
for(size_t i = 0; i < queries.size(); i++) {
- auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("dummy_sorting_field", "DESC")}, 2, 10, 1,
- MAX_SCORE, true, 10, spp::sparse_hash_set<std::string>(), {"p"});
+ auto results_op = collection->search(queries[i], search_fields, "", { }, {sort_by("dummy_sorting_field", "DESC")}, {2}, 10, 1,
+ MAX_SCORE, {true}, 10, spp::sparse_hash_set<std::string>(), {"p"});
if(results_op.ok() != true) {
exit(2);
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 85f4fafc..786211f1 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -169,8 +169,6 @@ TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {
auto results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
1, FREQUENCY, {true, true}).get();
- LOG(INFO) << results;
-
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
@@ -200,11 +198,34 @@ TEST_F(CollectionSpecificTest, OrderMultiFieldFuzzyMatch) {
ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
- 1, FREQUENCY, {true, true}).get();
+ 1, FREQUENCY, {true, true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 1}).get();
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+ results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true, true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {2, 1}).get();
+
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ // use extreme weights to push title matching ahead
+
+ results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true, true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {10, 1}).get();
+
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
collectionManager.drop_collection("coll1");
}
@@ -556,3 +577,39 @@ TEST_F(CollectionSpecificTest, SingleCharMatchFullFieldHighlight) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("description", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "Foo bar baz";
+ doc1["description"] = "Share information with this device.";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["title"] = "Foo Random";
+ doc2["description"] = "The Bar Fox";
+ doc2["points"] = 250;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("foo bar", {"title", "description"}, "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {false, false},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {4, 1}).get();
+
+ LOG(INFO) << results;
+
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index e537b965..0e9221a2 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -859,7 +859,10 @@ TEST_F(CollectionTest, MultipleFields) {
// when "starring" takes higher priority than "title"
query_fields = {"starring", "title"};
- results = coll_mul_fields->search("thomas", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ results = coll_mul_fields->search("thomas", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {2, 1}).get();
ASSERT_EQ(4, results["hits"].size());
ids = {"15", "12", "13", "14"};
@@ -2925,6 +2928,17 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
"<mark>", "</mark>", {1, 4}).get();
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ // use same weights
+
+ results = coll1->search("on a jetplane",
+ {"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
+ {true}, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 1}).get();
+
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());