Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Exact matches should be ranked above other matches.

kishorenc1387f1c21b0c
diff --git a/include/match_score.h b/include/match_score.h
index 8cdcec07..5224c9dd 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -50,7 +50,7 @@ struct Match {
uint64_t match_score = (
(int64_t(words_present) << 16) |
(int64_t(255 - total_cost) << 8) |
- (int64_t(distance))
+ (int64_t(100 - distance))
);
return match_score;
@@ -61,7 +61,7 @@ struct Match {
uint64_t match_score = (
(int64_t(words_present) << 16) |
(int64_t(255 - total_cost) << 8) |
- (int64_t(distance))
+ (int64_t(100 - distance))
);
return match_score;
@@ -206,10 +206,8 @@ struct Match {
best_displacement = 0;
}
- uint8_t best_distance = uint8_t(100 - best_displacement);
-
words_present = best_num_match;
- distance = best_distance;
+ distance = uint8_t(best_displacement);
if(populate_window) {
offsets = best_window;
}
diff --git a/src/collection.cpp b/src/collection.cpp
index c5be2187..a98e0e1a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -579,10 +579,15 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
// process weights for search fields
std::vector<search_field_t> weighted_search_fields;
- if(query_by_weights.empty() && !search_fields.empty()) {
- for(int i=search_fields.size(); i > 0; i--) {
+ if(query_by_weights.empty()) {
+ for(size_t i=1; i <= search_fields.size(); i++) {
query_by_weights.push_back(i);
}
+ } else {
+ auto max_weight = *std::max_element(query_by_weights.begin(), query_by_weights.end());
+ for(size_t i=0; i < query_by_weights.size(); i++) {
+ query_by_weights[i] = (max_weight - query_by_weights[i]) + 1;
+ }
}
for(size_t i=0; i < search_fields.size(); i++) {
diff --git a/src/index.cpp b/src/index.cpp
index b2cf8566..c272eeec 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1636,6 +1636,7 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
all_result_ids = filter_ids;
filter_ids = nullptr;
} else {
+ // In multi-field searches, a record can be matched across different fields, so we use this for aggregation
spp::sparse_hash_map<uint64_t, std::vector<KV*>> topster_ids;
std::vector<token_t> q_include_pos_tokens;
@@ -1700,48 +1701,42 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
}
}
- for(const auto& key_kvs: topster_ids) {
- // first calculate existing aggregate scores across best matching fields
- spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;
-
- const auto& kvs = key_kvs.second;
- const uint64_t seq_id = key_kvs.first;
+ for(const auto& seq_id_kvs: topster_ids) {
+ const uint64_t seq_id = seq_id_kvs.first;
+ const auto& kvs = seq_id_kvs.second; // each `kv` can be from a different field
// LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index];
- /*if(seq_id == 12 || seq_id == 15) {
- LOG(INFO) << "here";
- }*/
-
+ // to calculate existing aggregate scores across best matching fields
+ spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;
for(const auto kv: kvs) {
existing_field_kvs.emplace(kv->field_id, kv);
}
- uint64_t aggregated_score = 0;
uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
+ uint64_t total_typos = 0, total_distances = 0;
//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
- for(size_t i = 0; i < num_search_fields && num_search_fields > 1; i++) {
+ for(size_t i = 0; i < num_search_fields; i++) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
size_t weight = search_fields[i].weight;
- //LOG(INFO) << "--- field index: " << i;
-
- if(field_id == kvs[0]->field_id) {
- token_bits |= kvs[0]->token_bits;
- //LOG(INFO) << "kvs[0]->field_id pop count: " << __builtin_popcount(token_bits);
- aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] * weight);
- continue;
- }
+ //LOG(INFO) << "--- field index: " << i << ", weight: " << weight;
if(existing_field_kvs.count(field_id) != 0) {
// for existing field, we will simply sum field-wise weighted scores
token_bits |= existing_field_kvs[field_id]->token_bits;
//LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);
- aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]
- * weight);
+ int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];
+ total_distances += ((100 - (match_score & 0xFF)) + 1) * weight;
+ total_typos += ((255 - ((match_score >> 8) & 0xFF)) + 1) * weight;
+
+ /*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))
+ << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * weight
+ << ", total dist: " << (((match_score & 0xFF)))
+ << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * weight;*/
continue;
}
@@ -1780,14 +1775,25 @@ void Index::search(const std::vector<std::string>& q_include_tokens,
}
if(words_present != 0) {
- uint64_t match_score = Match::get_match_score(words_present, 0, 100);
- aggregated_score += (match_score * weight);
+ uint64_t match_score = Match::get_match_score(words_present, 0, 0);
+ total_distances += ((100 - (match_score & 0xFF)) + 1) * weight;
+ total_typos += ((255 - ((match_score >> 8) & 0xFF)) + 1) * weight;
+ //LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);
}
}
- aggregated_score |= (uint64_t(__builtin_popcount(token_bits)) << 48);
+ int64_t tokens_present = int64_t(__builtin_popcount(token_bits)) - 1;
+ total_typos = std::min<uint64_t>(255, total_typos);
+ total_distances = std::min<uint64_t>(100, total_distances);
+
+ uint64_t aggregated_score = (
+ (tokens_present << 16) |
+ ((255 - total_typos) << 8) |
+ (100 - total_distances)
+ );
- /*LOG(INFO) << "seq id: " << seq_id << ", pop count: " << __builtin_popcount(token_bits)
+ /*LOG(INFO) << "seq id: " << seq_id << ", tokens_present: " << tokens_present
+ << ", total_distances: " << total_distances << ", total_typos: " << total_typos
<< ", aggregated_score: " << aggregated_score << ", token_bits: " << token_bits;*/
kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;
@@ -2154,6 +2160,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
groups_processed.emplace(distinct_id);
}
+ //LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;
KV kv(field_id, query_index, token_bits, seq_id, distinct_id, match_score_index, scores);
topster->add(&kv);
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 448a7850..49428829 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -859,8 +859,6 @@ TEST_F(CollectionTest, MultipleFields) {
query_fields = {"title", "starring"};
std::vector<std::string> facets;
- auto x = coll_mul_fields->search("Will", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false);
-
nlohmann::json results = coll_mul_fields->search("Will", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(4, results["hits"].size());
@@ -2901,6 +2899,61 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ results = coll1->search("swift",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 1}).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionTest, MultiFieldRelevance4) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"Madras Dreams", "Chennai King"},
+ {"Madurai Express", "Madura Maddy"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("madras",
+ {"title", "artist"}, "", {}, {}, 2, 10, 1, FREQUENCY,
+ true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1, 1}).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
collectionManager.drop_collection("coll1");
}
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index ae6a1047..0be9931a 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -23,7 +23,7 @@ TEST(MatchTest, MatchScoreV2) {
auto match = Match(100, token_offsets, true);
ASSERT_EQ(4, match.words_present);
- ASSERT_EQ(97, match.distance);
+ ASSERT_EQ(3, match.distance);
std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -33,7 +33,7 @@ TEST(MatchTest, MatchScoreV2) {
// without populate window
match = Match(100, token_offsets, false);
ASSERT_EQ(4, match.words_present);
- ASSERT_EQ(97, match.distance);
+ ASSERT_EQ(3, match.distance);
ASSERT_EQ(0, match.offsets.size());
token_offsets.clear();
@@ -43,7 +43,7 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true);
ASSERT_EQ(3, match.words_present);
- ASSERT_EQ(98, match.distance);
+ ASSERT_EQ(2, match.distance);
expected_offsets = {170, 171, 169};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -57,7 +57,7 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true);
ASSERT_EQ(1, match.words_present);
- ASSERT_EQ(100, match.distance);
+ ASSERT_EQ(0, match.distance);
expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -67,7 +67,7 @@ TEST(MatchTest, MatchScoreV2) {
// without populate window
match = Match(100, token_offsets, false);
ASSERT_EQ(1, match.words_present);
- ASSERT_EQ(100, match.distance);
+ ASSERT_EQ(0, match.distance);
ASSERT_EQ(0, match.offsets.size());