Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Split-up fuzzy lookup into separate stages.

Kishore Nallan30cd057201b3
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c71afb6a..2d1bff4e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 2.8)
project(search)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O2")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O3")
include_directories(include)
include_directories(external/for)
diff --git a/include/art.h b/include/art.h
index 59d894ca..14c2b0f5 100644
--- a/include/art.h
+++ b/include/art.h
@@ -243,6 +243,8 @@ int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, ar
*/
int art_iter_fuzzy_prefix(art_tree *t, const unsigned char *term, int term_len, int max_cost, int max_words, std::vector<art_leaf*> & results);
+static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/art.cpp b/src/art.cpp
index 6a534af8..4c6087d1 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -28,7 +28,27 @@
* We order the leaves descending based on the score.
*/
bool compare_art_leaf(const art_leaf* a, const art_leaf* b) {
- return a->max_score > b->max_score;
+ return a->token_count > b->token_count;
+}
+
+bool compare_art_node(const art_node* a, const art_node* b) {
+ uint32_t a_score = 0, b_score = 0;
+
+ if(IS_LEAF(a)) {
+ art_leaf* al = (art_leaf *) LEAF_RAW(a);
+ a_score = al->token_count;
+ } else {
+ a_score = a->max_token_count;
+ }
+
+ if(IS_LEAF(b)) {
+ art_leaf* bl = (art_leaf *) LEAF_RAW(b);
+ b_score = bl->token_count;
+ } else {
+ b_score = b->max_token_count;
+ }
+
+ return a_score > b_score;
}
/**
@@ -54,6 +74,8 @@ static art_node* alloc_node(uint8_t type) {
abort();
}
n->type = type;
+ n->max_score = 0;
+ n->max_token_count = 0;
return n;
}
@@ -250,7 +272,7 @@ void* art_search(const art_tree *t, const unsigned char *key, int key_len) {
n = (art_node *) LEAF_RAW(n);
// Check if the expanded path matches
if (!leaf_matches((art_leaf*)n, key, key_len, depth)) {
- return ((art_leaf*)n)->values;
+ return ((art_leaf*)n);
}
return NULL;
}
@@ -353,9 +375,11 @@ static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {
static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);
l->values = new art_values;
- add_document_to_leaf(document, l);
+ l->token_count = 0;
+ l->max_score = 0;
l->key_len = key_len;
memcpy(l->key, key, key_len);
+ add_document_to_leaf(document, l);
return l;
}
@@ -571,7 +595,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
}
n->max_score = (uint16_t) MAX(n->max_score, (const uint16_t &) document->score);
- n->max_token_count = (uint16_t) MAX(n->max_token_count, num_hits);
+ n->max_token_count = MAX(n->max_token_count, num_hits);
// Check if given node has a prefix
if (n->partial_len) {
@@ -809,41 +833,41 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {
return NULL;
}
+static uint32_t get_score(art_node* child) {
+ if (IS_LEAF(child)) {
+ art_leaf *l = (art_leaf *) LEAF_RAW(child);
+ return l->token_count;
+ }
+
+ return child->max_token_count;
+}
+
static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {
printf("INSIDE topk_iter: root->type: %d\n", root->type);
- // FIXME: try to avoid branching
auto cmp = [term_len](art_node * left, art_node * right) {
int lscore, rscore;
if (IS_LEAF(left)) {
art_leaf *l = (art_leaf *) LEAF_RAW(left);
- // Exact matches should be boosted ahead of others
- if(l->key_len == term_len) {
- lscore = std::numeric_limits<uint16_t>::max();
- } else {
- lscore = l->max_score;
- }
+ lscore = l->token_count;
} else {
- lscore = left->max_score;
+ lscore = left->max_token_count;
}
if(IS_LEAF(right)) {
art_leaf *r = (art_leaf *) LEAF_RAW(right);
- if(r->key_len == term_len) {
- rscore = std::numeric_limits<uint16_t>::max();
- } else {
- rscore = r->max_score;
- }
+ rscore = r->token_count;
} else {
- rscore = right->max_score;
+ rscore = right->max_token_count;
}
+ // priority queue sorts based on priority (so use < for sorting descending order by score)
return lscore < rscore;
};
+
std::priority_queue<art_node *, std::vector<art_node *>, decltype(cmp)> q(cmp);
q.push(root);
- //std::cout << "ROOT SCORE: " << root->score << std::endl;
while(!q.empty() && results.size() < k) {
art_node *n = (art_node *) q.top();
@@ -852,7 +876,8 @@ static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*>
if (!n) continue;
if (IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
- //std::cout << "\nLEAF, SCORE: " << l->score << std::endl;
+ //printf("\nLEAF: %.*s", leaf->key_len, leaf->key);
+ //std::cout << ", SCORE: " << l->token_count << std::endl;
results.push_back(l);
continue;
}
@@ -860,38 +885,33 @@ static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*>
int idx;
switch (n->type) {
case NODE4:
- //std::cout << "\nNODE4, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE4, SCORE: " << n->max_token_count << std::endl;
for (int i=0; i < n->num_children; i++) {
art_node* child = ((art_node4*)n)->children[i];
- if(IS_LEAF(child)) {
- art_leaf* l = (art_leaf*)LEAF_RAW(child);
- //std::cout << "LSCORE: " << l->score << std::endl;
- } else {
- //std::cout << "SCORE: " << child->score << std::endl;
- }
q.push(child);
}
- //std::cout << "---" << std::endl;
break;
case NODE16:
- //std::cout << "\nNODE16, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE16, SCORE: " << n->max_token_count << std::endl;
for (int i=0; i < n->num_children; i++) {
q.push(((art_node16*)n)->children[i]);
}
break;
case NODE48:
- //std::cout << "\nNODE48, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE48, SCORE: " << n->max_token_count << std::endl;
for (int i=0; i < 256; i++) {
idx = ((art_node48*)n)->keys[i];
if (!idx) continue;
- q.push(((art_node48*)n)->children[idx-1]);
+ art_node *child = ((art_node48*)n)->children[idx - 1];
+ //std::cout << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child) << std::endl;
+ q.push(child);
}
break;
case NODE256:
- //std::cout << "\nNODE256, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE256, SCORE: " << n->max_token_count << std::endl;
for (int i=0; i < 256; i++) {
if (!((art_node256*)n)->children[i]) continue;
q.push(((art_node256*)n)->children[i]);
@@ -1072,7 +1092,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call
/* reached end of term, and cost is below threshold, print children of this node as matches*/\
if(new_current_row[term_len] <= max_cost) {\
printf("START RECURSIVE ITER\n");\
- topk_iter(child, term_len, max_words, results);\
+ results.push_back(child);\
}\
} else if(row_min <= max_cost) {\
int new_depth = (child_char != 0) ? depth+1 : depth;\
@@ -1089,7 +1109,7 @@ void print_row(int* row, int row_len) {
}
int levenshtein_score(char ch, const unsigned char* term, const int term_len, int* previous_row, int* current_row) {
- int row_min = 12;
+ int row_min = std::numeric_limits<int>::max();
int insert_or_del, replace;
printf("\nPREVIOUS ROW: ");
@@ -1116,7 +1136,7 @@ void copyIntArray(int *dest, int *src, int len) {
}
static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term, int term_len, int max_cost, int max_words,
- int depth, int* previous_row, std::vector<art_leaf*> & results) {
+ int depth, int* previous_row, std::vector<art_node*> & results) {
if (!n) return 0;
if (IS_LEAF(n)) {
@@ -1147,7 +1167,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
}
if(current_row[term_len] <= max_cost) {
- topk_iter(n, term_len, max_words, results);
+ results.push_back(n);
}
return 0;
@@ -1179,7 +1199,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
if(depth == term_len) {
if(current_row[term_len] <= max_cost) {
printf("PARTIAL START RECURSIVE ITER\n");
- return topk_iter(n, term_len, max_words, results);
+ results.push_back(n);
}
return 0;
@@ -1261,5 +1281,13 @@ int art_iter_fuzzy_prefix(art_tree *t, const unsigned char *term, int term_len,
previous_row[i] = i;
}
- return art_iter_fuzzy_prefix_recurse(t->root, term, term_len, max_cost, max_words, 0, previous_row, results);
+ std::vector<art_node*> nodes;
+ art_iter_fuzzy_prefix_recurse(t->root, term, term_len, max_cost, max_words, 0, previous_row, nodes);
+ std::sort(nodes.begin(), nodes.end(), compare_art_node);
+
+ for(auto node: nodes) {
+ topk_iter(node, term_len, max_words, results);
+ }
+
+ return 0;
}
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
index 36398101..34ab5eda 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -79,9 +79,9 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_
document.offsets = new uint32_t[kv.second.size()];
uint32_t num_hits = document.offsets_len;
- art_values* values = (art_values *) art_search(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length());
- if(values != NULL) {
- num_hits += values->ids.getLength();
+ art_leaf* leaf = (art_leaf *) art_search(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length());
+ if(leaf != NULL) {
+ num_hits += leaf->token_count;
}
for(auto i=0; i<kv.second.size(); i++) {
@@ -110,8 +110,11 @@ void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores,
for(auto token: tokens) {
for(int max_cost=0; max_cost<2; max_cost++) {
vector<art_leaf*> leaves;
- art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), max_cost, 3, leaves);
+ art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), max_cost, 20, leaves);
if(!leaves.empty()) {
+ for(auto i=0; i<leaves.size(); i++) {
+ cout << leaves[i]->key << " / " << leaves[i]->token_count << endl;
+ }
token_leaves.push_back(leaves);
break;
}
@@ -176,7 +179,8 @@ void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores,
MatchScore mscore = match_score(doc_id, token_positions);
const uint32_t cumulativeScore = ((uint32_t)(mscore.words_present * 16 + (20 - mscore.distance)) * 64000) + docscores[doc_id];
- //cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << mscore.words_present << endl;
+// cout << "result_ids[i]: " << result_ids[i] << " - mscore.distance: " << (int)mscore.distance << " - mscore.words_present: " << (int)mscore.words_present
+// << " - docscores[doc_id]: " << (int)docscores[doc_id] << " - cumulativeScore: " << cumulativeScore << endl;
topster.add(doc_id, cumulativeScore);
}
@@ -213,7 +217,7 @@ int main() {
unordered_map<uint32_t, uint16_t> docscores;
- //std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");
+// std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");
std::ifstream infile("/data/hnstories.tsv");
std::string line;
@@ -257,19 +261,20 @@ int main() {
std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;
}*/
- find_documents(t, docscores, "app;e", 10);
+ ///find_documents(t, docscores, "app;e", 10);
- /*string token = "lanch";
+ string token = "app;e";
vector<art_leaf*> leaves;
auto begin = std::chrono::high_resolution_clock::now();
- art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), 1, 4, leaves);
+ art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), 1, 10, leaves);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
for(auto leaf: leaves) {
- cout << "Word: " << leaf->key << " - score: " << leaf->max_score << endl;
+ printf("Word: %.*s", leaf->key_len, leaf->key);
+ cout << " - score: " << leaf->token_count << endl;
}
- cout << "Time taken: " << timeMillis << "us" << endl;*/
+ cout << "Time taken: " << timeMillis << "us" << endl;
art_tree_destroy(&t);
return 0;