Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Weight all components of cross-field match score.

Kishore Nallan48ac6bb82a87
diff --git a/src/index.cpp b/src/index.cpp
index 8a75f6b5..829647e5 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1716,10 +1716,10 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
uint64_t total_typos = 0, total_distances = 0, min_typos = 1000;
- uint64_t verbatim_match_fields = 0; // query matching field verbatim
- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens
+ uint64_t verbatim_match_fields = 0; // query matching field verbatim
+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens
uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field
- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)
+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)
//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
@@ -1747,13 +1747,13 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;
int64_t exact_match_score = match_score & 0xFF;
- verbatim_match_fields += exact_match_score;
+ verbatim_match_fields += (weight * exact_match_score);
uint64_t unique_tokens_found =
int64_t(__builtin_popcount(existing_field_kvs[field_id]->token_bits)) - 1;
if(field_typos == 0 && unique_tokens_found == field_query_tokens[i].q_include_tokens.size()) {
- exact_match_fields++;
+ exact_match_fields += weight;
}
auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);
@@ -1765,7 +1765,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
min_typos = field_typos;
}
- total_token_matches += tokens_found;
+ total_token_matches += (weight * tokens_found);
/*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))
<< ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * priority
@@ -1817,8 +1817,8 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
total_typos += (field_typos + 1) * priority;
if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {
- exact_match_fields++;
- verbatim_match_fields++; // this is only an approximate
+ exact_match_fields += weight;
+ verbatim_match_fields += weight; // this is only an approximate
}
auto weighted_tokens_match = (tokens_found * weight) + (MAX_SUM_TYPOS - field_typos + 1);
@@ -1831,7 +1831,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
min_typos = field_typos;
}
- total_token_matches += tokens_found;
+ total_token_matches += (weight * tokens_found);
//LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);
}
}
@@ -1839,9 +1839,11 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
// num tokens present across fields including those containing typos
int64_t uniq_tokens_found = int64_t(__builtin_popcount(token_bits)) - 1;
+ verbatim_match_fields = std::min<uint64_t>(255, verbatim_match_fields);
+ exact_match_fields = std::min<uint64_t>(255, exact_match_fields);
+ max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);
total_typos = std::min<uint64_t>(255, total_typos);
total_distances = std::min<uint64_t>(100, total_distances);
- max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);
uint64_t aggregated_score = (
(verbatim_match_fields << 56) | // field value *exactly* same as query tokens
@@ -1857,6 +1859,7 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
//LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;
/*LOG(INFO) << "seq id: " << seq_id
+ << ", verbatim_match_fields: " << verbatim_match_fields
<< ", exact_match_fields: " << exact_match_fields
<< ", max_weighted_tokens_match: " << max_weighted_tokens_match
<< ", uniq_tokens_found: " << uniq_tokens_found
@@ -1864,7 +1867,6 @@ void Index::search(const std::vector<query_tokens_t>& field_query_tokens,
<< ", total_token_matches: " << total_token_matches
<< ", typo score: " << (255 - total_typos)
<< ", distance score: " << (100 - total_distances)
- << ", verbatim_match_fields: " << verbatim_match_fields
<< ", aggregated_score: " << aggregated_score << ", token_bits: " << token_bits;*/
kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 41006252..7a98888c 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1279,5 +1279,48 @@ TEST_F(CollectionSpecificTest, TypoCorrectionWithFaceting) {
ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, MultiFieldMatchesShouldBeWeighted) {
+ // 2 matches on low weighted fields should not overpower a single match on high weighted field
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, false),
+ field("label", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Twin";
+ doc1["category"] = "kids";
+ doc1["label"] = "kids";
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "Kids";
+ doc2["category"] = "children";
+ doc2["label"] = "children";
+ doc2["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("kids", {"name", "category", "label"},
+ "", {}, {}, {0, 0, 0}, 10,
+ 1, FREQUENCY, {false, false, false},
+ 2, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {6, 1, 1}).get();
+
+ LOG(INFO) << results;
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index e5a62584..5bef8283 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -3609,7 +3609,10 @@ TEST_F(CollectionTest, MultiFieldMatchRankingOnFieldOrder) {
}
auto results = coll1->search("michael jackson toxic",
- {"title", "artist"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5).get();
+ {"title", "artist"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 6}).get();
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());