Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor bucket text match logic + add test.
Kishore Nallan80a4e7d17ddc
diff --git a/src/collection.cpp b/src/collection.cppindex 330ce4fe..27a30716 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1067,11 +1067,18 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sstd::vector<int64_t> result_scores(max_kvs_bucketed);// only first `max_kvs_bucketed` elements are bucketed to prevent pagination issues past 250 records- for(size_t i = 0; i < max_kvs_bucketed; i++) {- result_scores[i] = raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index];- size_t block_index = (i / num_buckets) * num_buckets;- int64_t text_match = raw_result_kvs[block_index][0]->scores[raw_result_kvs[block_index][0]->match_score_index];- raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index] = text_match;+ size_t block_len = (max_kvs_bucketed < num_buckets) ? max_kvs_bucketed : (max_kvs_bucketed / num_buckets) + 1;+ size_t i = 0;+ while(i < max_kvs_bucketed) {+ int64_t anchor_score = raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index];+ size_t j = 0;+ while(j < block_len && i+j < max_kvs_bucketed) {+ result_scores[i+j] = raw_result_kvs[i+j][0]->scores[raw_result_kvs[i+j][0]->match_score_index];+ raw_result_kvs[i+j][0]->scores[raw_result_kvs[i+j][0]->match_score_index] = anchor_score;+ j++;+ }++ i += j;}// sort again based on bucketed match scorediff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 163f48e1..e7543d57 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -1369,6 +1369,48 @@ TEST_F(CollectionSortingTest, TextMatchBucketRanking) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionSortingTest, TextMatchMoreDocsThanBuckets) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ std::vector<std::vector<std::string>> records = {+ {"Mark Antony"},+ {"Marks Spencer"},+ {"Marking Rhine"},+ {"Markolm Spane"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["points"] = i;+ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ sort_fields = {+ sort_by("_text_match(buckets: 2)", "DESC"),+ sort_by("points", "DESC"),+ };++ auto results = coll1->search("mark", {"title"},+ "", {}, sort_fields, {0}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {1}, 1000, true).get();++ ASSERT_EQ(4, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("2", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][2]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionSortingTest, RepeatingTokenRanking) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};