Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Move to precise token candidate selection.

Kishore Nallan19badcd0cbaa
diff --git a/include/index.h b/include/index.h
index 159b2ad2..992f6ac6 100644
--- a/include/index.h
+++ b/include/index.h
@@ -314,7 +314,7 @@ public:
enum {FIELD_LIMIT_NUM = 100};
enum {COMBINATION_MAX_LIMIT = 10000};
- enum {COMBINATION_MIN_LIMIT = 10};
+ enum {COMBINATION_MIN_LIMIT = 4};
// If the number of results found is less than this threshold, Typesense will attempt to drop the tokens
// in the query that have the least individual hits one by one until enough results are found.
diff --git a/src/art.cpp b/src/art.cpp
index ca5534f1..d6726b5a 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -886,18 +886,12 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
const std::set<art_leaf*>& exclude_leaves, std::vector<art_leaf *> &results) {
printf("INSIDE art_topk_iter: root->type: %d\n", root->type);
- std::priority_queue<const art_node *, std::vector<const art_node *>,
- decltype(&compare_art_node_score_pq)> q(compare_art_node_score_pq);
-
- if(token_order == FREQUENCY) {
- q = std::priority_queue<const art_node *, std::vector<const art_node *>,
- decltype(&compare_art_node_frequency_pq)>(compare_art_node_frequency_pq);
- }
+ std::queue<const art_node *> q;
q.push(root);
- while(!q.empty() && results.size() < max_results) {
- art_node *n = (art_node *) q.top();
+ while(!q.empty() && results.size() < 10000) {
+ art_node *n = (art_node *) q.front();
q.pop();
/*if (IS_LEAF(n)) {
@@ -1446,16 +1440,22 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
//long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
- //auto begin = std::chrono::high_resolution_clock::now();
+ auto begin = std::chrono::high_resolution_clock::now();
for(auto node: nodes) {
art_topk_iter(node, token_order, max_words, filter_ids, filter_ids_length, exclude_leaves, results);
}
+ size_t limit_len = std::min<size_t>(max_words, results.size());
+
if(token_order == FREQUENCY) {
- std::sort(results.begin(), results.end(), compare_art_leaf_frequency);
+ std::partial_sort(results.begin(), results.begin() + limit_len, results.end(), compare_art_leaf_frequency);
} else {
- std::sort(results.begin(), results.end(), compare_art_leaf_score);
+ std::partial_sort(results.begin(), results.begin() + limit_len, results.end(), compare_art_leaf_score);
+ }
+
+ if(results.size() > max_words) {
+ results.resize(max_words);
}
/*auto time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 5b3419fd..2784fbe7 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -683,8 +683,8 @@ TEST(ArtTest, test_art_fuzzy_search) {
art_fuzzy_search(&t, (const unsigned char *) "pltinum", strlen("pltinum"), 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);
ASSERT_EQ(2, leaves.size());
- ASSERT_STREQ("platinumsmith", (const char *)leaves.at(0)->key);
- ASSERT_STREQ("platinum", (const char *)leaves.at(1)->key);
+ ASSERT_STREQ("platinum", (const char *)leaves.at(0)->key);
+ ASSERT_STREQ("platinumsmith", (const char *)leaves.at(1)->key);
leaves.clear();
@@ -734,7 +734,7 @@ TEST(ArtTest, test_art_fuzzy_search) {
art_fuzzy_search(&t, (const unsigned char *) "hown", strlen("hown") + 1, 0, 1, 10, FREQUENCY, false, nullptr, 0, leaves);
ASSERT_EQ(10, leaves.size());
- std::vector<const char*> words = {"town", "sown", "shown", "own", "mown", "lown", "howl", "howk", "howe", "how"};
+ std::vector<const char*> words = {"lown", "how", "shown", "howl", "howe", "mown", "town", "sown", "own", "howk"};
for(size_t leaf_index = 0; leaf_index < leaves.size(); leaf_index++) {
ASSERT_STREQ(words.at(leaf_index), (const char *)leaves.at(leaf_index)->key);
}
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index aecdfc4c..540748de 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -181,7 +181,7 @@ TEST_F(CollectionSortingTest, NoDefaultSortingField) {
results = coll1->search("*", {}, "", {}, {}, {1}, 30, 1, FREQUENCY, {false}).get();
- ASSERT_EQ(23, results["found"]);
+ ASSERT_EQ(23, results["found"].get<size_t>());
ASSERT_EQ(23, results["hits"].size());
ASSERT_EQ(23, results["out_of"]);
@@ -202,7 +202,7 @@ TEST_F(CollectionSortingTest, FrequencyOrderedTokensWithoutDefaultSortingField)
coll1 = collectionManager.create_collection("coll1", 1, fields).get();
}
- // since only top 10 tokens are fetched for prefixes, the "end" should not show up in the results
+ // since only top 4 tokens are fetched for prefixes, the "enyzme" should not show up in the results
std::vector<std::string> tokens = {
"enter", "elephant", "enamel", "ercot", "enyzme", "energy",
"epoch", "epyc", "express", "everest", "end"
@@ -223,13 +223,13 @@ TEST_F(CollectionSortingTest, FrequencyOrderedTokensWithoutDefaultSortingField)
auto results = coll1->search("e", {"title"}, "", {}, {}, {0}, 100, 1, NOT_SET, {true}).get();
- // 11 + 10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2
- ASSERT_EQ(65, results["found"]);
+ // [11 + 10 + 9 + 8] + 7 + 6 + 5 + 4 + 3 + 2
+ ASSERT_EQ(38, results["found"].get<size_t>());
// we have to ensure that no result contains the word "end" since it occurs least number of times
bool found_end = false;
for(auto& res: results["hits"].items()) {
- if(res.value()["document"]["title"] == "end") {
+ if(res.value()["document"]["title"] == "enyzme") {
found_end = true;
}
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 3891bd04..8ce03d7c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -638,7 +638,7 @@ TEST_F(CollectionTest, PrefixSearching) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
"", 10).get();
ASSERT_EQ(2, results["hits"].size());
- ids = {"19", "22"};
+ ids = {"1", "2"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
@@ -1035,7 +1035,7 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {
FREQUENCY, {true}, 1000, empty, empty, 10).get();
ASSERT_EQ(3, results["hits"].size());
- ASSERT_EQ(7, results["found"].get<int>());
+ ASSERT_EQ(6, results["found"].get<int>());
// cannot fetch more than in-built limit of 250
auto res_op = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 251, 1,
@@ -1057,19 +1057,13 @@ TEST_F(CollectionTest, KeywordQueryReturnsResultsBasedOnPerPageParam) {
FREQUENCY, {true}, 1000, empty, empty, 10).get();
ASSERT_EQ(3, results["hits"].size());
- ASSERT_EQ(7, results["found"].get<int>());
+ ASSERT_EQ(6, results["found"].get<int>());
results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 3, 2,
FREQUENCY, {true}, 1000, empty, empty, 10).get();
ASSERT_EQ(3, results["hits"].size());
- ASSERT_EQ(7, results["found"].get<int>());
-
- results = coll_mul_fields->search("w", query_fields, "", facets, sort_fields, {0}, 3, 3,
- FREQUENCY, {true}, 1000, empty, empty, 10).get();
-
- ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(7, results["found"].get<int>());
+ ASSERT_EQ(6, results["found"].get<int>());
collectionManager.drop_collection("coll_mul_fields");
}