Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor bucket text match logic + add test.

Kishore Nallan80a4e7d17ddc
diff --git a/src/collection.cpp b/src/collection.cpp
index 330ce4fe..27a30716 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1067,11 +1067,18 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::vector<int64_t> result_scores(max_kvs_bucketed);
// only first `max_kvs_bucketed` elements are bucketed to prevent pagination issues past 250 records
- for(size_t i = 0; i < max_kvs_bucketed; i++) {
- result_scores[i] = raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index];
- size_t block_index = (i / num_buckets) * num_buckets;
- int64_t text_match = raw_result_kvs[block_index][0]->scores[raw_result_kvs[block_index][0]->match_score_index];
- raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index] = text_match;
+ size_t block_len = (max_kvs_bucketed < num_buckets) ? max_kvs_bucketed : (max_kvs_bucketed / num_buckets) + 1;
+ size_t i = 0;
+ while(i < max_kvs_bucketed) {
+ int64_t anchor_score = raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index];
+ size_t j = 0;
+ while(j < block_len && i+j < max_kvs_bucketed) {
+ result_scores[i+j] = raw_result_kvs[i+j][0]->scores[raw_result_kvs[i+j][0]->match_score_index];
+ raw_result_kvs[i+j][0]->scores[raw_result_kvs[i+j][0]->match_score_index] = anchor_score;
+ j++;
+ }
+
+ i += j;
}
// sort again based on bucketed match score
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 163f48e1..e7543d57 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -1369,6 +1369,48 @@ TEST_F(CollectionSortingTest, TextMatchBucketRanking) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSortingTest, TextMatchMoreDocsThanBuckets) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"Mark Antony"},
+ {"Marks Spencer"},
+ {"Marking Rhine"},
+ {"Markolm Spane"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["points"] = i;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ sort_fields = {
+ sort_by("_text_match(buckets: 2)", "DESC"),
+ sort_by("points", "DESC"),
+ };
+
+ auto results = coll1->search("mark", {"title"},
+ "", {}, sort_fields, {0}, 10,
+ 1, FREQUENCY, {true},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {1}, 1000, true).get();
+
+ ASSERT_EQ(4, results["hits"].size());
+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionSortingTest, RepeatingTokenRanking) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};