Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Move to precise token candidate selection.
Kishore Nallan19badcd0cbaa
diff --git a/include/index.h b/include/index.hindex 159b2ad2..992f6ac6 100644--- a/include/index.h+++ b/include/index.h@@ -314,7 +314,7 @@ public:enum {FIELD_LIMIT_NUM = 100};enum {COMBINATION_MAX_LIMIT = 10000};- enum {COMBINATION_MIN_LIMIT = 10};+ enum {COMBINATION_MIN_LIMIT = 4};// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens// in the query that have the least individual hits one by one until enough results are found.diff --git a/src/art.cpp b/src/art.cppindex ca5534f1..d6726b5a 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -886,18 +886,12 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rconst std::set<art_leaf*>& exclude_leaves, std::vector<art_leaf *> &results) {printf("INSIDE art_topk_iter: root->type: %d\n", root->type);- std::priority_queue<const art_node *, std::vector<const art_node *>,- decltype(&compare_art_node_score_pq)> q(compare_art_node_score_pq);-- if(token_order == FREQUENCY) {- q = std::priority_queue<const art_node *, std::vector<const art_node *>,- decltype(&compare_art_node_frequency_pq)>(compare_art_node_frequency_pq);- }+ std::queue<const art_node *> q;q.push(root);- while(!q.empty() && results.size() < max_results) {- art_node *n = (art_node *) q.top();+ while(!q.empty() && results.size() < 10000) {+ art_node *n = (art_node *) q.front();q.pop();/*if (IS_LEAF(n)) {@@ -1446,16 +1440,22 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,//long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();- //auto begin = std::chrono::high_resolution_clock::now();+ auto begin = std::chrono::high_resolution_clock::now();for(auto node: nodes) {art_topk_iter(node, token_order, max_words, filter_ids, filter_ids_length, exclude_leaves, results);}+ size_t limit_len = std::min<size_t>(max_words, results.size());+if(token_order == FREQUENCY) {- std::sort(results.begin(), results.end(), compare_art_leaf_frequency);+ std::partial_sort(results.begin(), results.begin() + limit_len, results.end(), compare_art_leaf_frequency);} else {- std::sort(results.begin(), results.end(), compare_art_leaf_score);+ std::partial_sort(results.begin(), results.begin() + limit_len, results.end(), compare_art_leaf_score);+ }++ if(results.size() > max_words) {+ results.resize(max_words);}/*auto time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/test/art_test.cpp b/test/art_test.cppindex 5b3419fd..2784fbe7 100644--- a/test/art_test.cpp+++ b/test/art_test.cpp@@ -683,8 +683,8 @@ TEST(ArtTest, test_art_fuzzy_search) {art_fuzzy_search(&t, (const unsigned char *) "pltinum", strlen("pltinum"), 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);ASSERT_EQ(2, leaves.size());- ASSERT_STREQ("platinumsmith", (const char *)leaves.at(0)->key);- ASSERT_STREQ("platinum", (const char *)leaves.at(1)->key);+ ASSERT_STREQ("platinum", (const char *)leaves.at(0)->key);+ ASSERT_STREQ("platinumsmith", (const char *)leaves.at(1)->key);leaves.clear();@@ -734,7 +734,7 @@ TEST(ArtTest, test_art_fuzzy_search) {art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, nullptr, 0, leaves);ASSERT_EQ(10, leaves.size());- std::vector<const char*> words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};+ std::vector<const char*> words = {"lown", "how", "shown", "howl", "howe", "mown", "town", "sown", "own", "howk"};for(size_t leaf_index = 0; leaf_index < leaves.size(); leaf_index++) {ASSERT_STREQ(words.at(leaf_index), (const char *)leaves.at(leaf_index)->key);}diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex aecdfc4c..540748de 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -181,7 +181,7 @@ TEST_F(CollectionSortingTest, NoDefaultSortingField) {results = coll1->search("*", {}, "", {}, {}, {1}, 30, 1, FREQUENCY, {false}).get();- ASSERT_EQ(23, results["found"]);+ ASSERT_EQ(23, results["found"].get<size_t>());ASSERT_EQ(23, results["hits"].size());ASSERT_EQ(23, results["out_of"]);@@ -202,7 +202,7 @@ TEST_F(CollectionSortingTest, FrequencyOrderedTokensWithoutDefaultSortingField)coll1 = collectionManager.create_collection("coll1", 1, fields).get();}- // since only top 10 tokens are fetched for prefixes, the "end" should not show up in the results+ // since only top 4 tokens are fetched for prefixes, the "enyzme" should not show up in the resultsstd::vector<std::string> tokens = {"enter", "elephant", "enamel", "ercot", "enyzme", "energy","epoch", "epyc", "express", "everest", "end"@@ -223,13 +223,13 @@ TEST_F(CollectionSortingTest, FrequencyOrderedTokensWithoutDefaultSortingField)auto results = coll1->search("e", {"title"}, "", {}, {}, {0}, 100, 1, NOT_SET, {true}).get();- // 11 + 10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2- ASSERT_EQ(65, results["found"]);+ // [11 + 10 + 9 + 8] + 7 + 6 + 5 + 4 + 3 + 2+ ASSERT_EQ(38, results["found"].get<size_t>());// we have to ensure that no result contains the word "end" since it occurs least number of timesbool found_end = false;for(auto& res: results["hits"].items()) {- if(res.value()["document"]["title"] == "end") {+ if(res.value()["document"]["title"] == "enyzme") {found_end = true;}}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 3891bd04..8ce03d7c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -638,7 +638,7 @@ TEST_F(CollectionTest, PrefixSearching) {spp::sparse_hash_set<std::string>(), 10, "", 30, 5,"", 10).get();ASSERT_EQ(2, results["hits"].size());- ids = {"19", "22"};+ ids = {"1", "2"};for(size_t i = 0; i < results["hits"].size(); i++) {nlohmann::json result = results["hits"].at(i);@@ -1035,7 +1035,7 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {FREQUENCY, {true}, 1000, empty, empty, 10).get();ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());+ ASSERT_EQ(6, results["found"].get<int>());// cannot fetch more than in-built limit of 250auto res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 251, 1,@@ -1057,19 +1057,13 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {FREQUENCY, {true}, 1000, empty, empty, 10).get();ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());+ ASSERT_EQ(6, results["found"].get<int>());results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 3, 2,FREQUENCY, {true}, 1000, empty, empty, 10).get();ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());-- results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 3, 3,- FREQUENCY, {true}, 1000, empty, empty, 10).get();-- ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(7, results["found"].get<int>());+ ASSERT_EQ(6, results["found"].get<int>());collectionManager.drop_collection("coll_mul_fields");}