Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Bug fix: word suggestions were not showing up sorted on their document scores.
Kishore Nallanb88241d9e965
diff --git a/include/art.h b/include/art.hindex 02f1ef39..2187b6da 100644--- a/include/art.h+++ b/include/art.h@@ -19,6 +19,8 @@ extern "C" {#define MAX_PREFIX_LEN 10+#define MAX(x, y) (((x) > (y)) ? (x) : (y))+#if defined(__GNUC__) && !defined(__clang__)# if __STDC_VERSION__ >= 199901L && 402 == (__GNUC__ * 100 + __GNUC_MINOR__)/*diff --git a/src/art.cpp b/src/art.cppindex 2db5061e..a427fbf9 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -22,25 +22,6 @@#define printf(fmt, ...) (0)#endif-static int levenshtein(const unsigned char *s1, unsigned int s1len, const unsigned char *s2, unsigned int s2len) {- unsigned int x, y, lastdiag, olddiag;- unsigned int column[s1len+1];- for (y = 1; y <= s1len; y++)- column[y] = y;- for (x = 1; x <= s2len; x++) {- column[0] = x;- for (y = 1, lastdiag = x-1; y <= s1len; y++) {- olddiag = column[y];- column[y] = MIN3(column[y] + 1, column[y-1] + 1, lastdiag + (s1[y-1] == s2[x-1] ? 0 : 1));- lastdiag = olddiag;- }- }-- //printf("s1: %s, s1len: %d, s2: %s, s2len: %d, diff: %d\n\n", s1, s1len, s2, s2len, column[s1len]);-- return(column[s1len]);-}-/*** Allocates a node of the given type,* initializes to zero and sets the type.@@ -351,7 +332,7 @@ static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_documart_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);l->values = new art_values;l->values->ids.append_sorted(document->id);- l->score = std::max(l->score, document->score);+ l->score = MAX(l->score, document->score); // somehow std::max does not seem to work here - always returns 0l->key_len = key_len;memcpy(l->key, key, key_len);return l;@@ -376,7 +357,7 @@ static void copy_header(art_node *dest, art_node *src) {static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child) {(void)ref;- n->n.score = std::max(n->n.score, ((art_leaf *) child)->score);+ n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);n->n.num_children++;n->children[c] = (art_node *) child;}@@ -385,7 +366,7 @@ static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *chif (n->n.num_children < 48) {int pos = 0;while (n->children[pos]) pos++;- n->n.score = std::max(n->n.score, ((art_leaf *) child)->score);+ n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);n->children[pos] = (art_node *) child;n->keys[c] = pos + 1;n->n.num_children++;@@ -426,7 +407,7 @@ static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *chidx = n->n.num_children;// Set the child- n->n.score = std::max(n->n.score, ((art_leaf *) child)->score);+ n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);n->keys[idx] = c;n->children[idx] = (art_node *) child;n->n.num_children++;@@ -461,9 +442,9 @@ static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *chil// Insert elementif (IS_LEAF(child)) {- n->n.score = std::max(n->n.score, ((art_leaf *) child)->score);+ n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);} else {- n->n.score = std::max(n->n.score, ((art_node *) child)->score);+ n->n.score = MAX(n->n.score, ((art_node *) child)->score);}n->keys[idx] = c;@@ -540,7 +521,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char **old = 1;art_values *old_val = l->values;l->values->ids.append_sorted(document->id);- l->score = std::max(l->score, document->score);+ l->score = MAX(l->score, document->score);return old_val;}@@ -553,7 +534,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Determine longest prefixuint32_t longest_prefix = longest_common_prefix(l, l2, depth);new_n->n.partial_len = longest_prefix;- new_n->n.score = (uint16_t) std::max(l->score, (const uint16_t &) document->score);;+ new_n->n.score = (uint16_t) MAX(l->score, (const uint16_t &) document->score);;memcpy(new_n->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));// Add the leafs to the new node4*ref = (art_node*)new_n;@@ -562,7 +543,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *return NULL;}- n->score = (uint16_t) std::max(n->score, (const uint16_t &) document->score);+ n->score = (uint16_t) MAX(n->score, (const uint16_t &) document->score);// Check if given node has a prefixif (n->partial_len) {@@ -577,7 +558,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *art_node4 *new_n = (art_node4*)alloc_node(NODE4);*ref = (art_node*)new_n;new_n->n.partial_len = prefix_diff;- new_n->n.score = (uint16_t) std::max(n->score, (const uint16_t &) document->score);+ new_n->n.score = (uint16_t) MAX(n->score, (const uint16_t &) document->score);memcpy(new_n->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff));// Adjust the prefix of the old node@@ -826,7 +807,7 @@ static int topk_iter(art_node *root, std::vector<art_leaf*> & results) {std::priority_queue<art_node *, std::vector<art_node *>, decltype(cmp)> q(cmp);q.push(root);- std::cout << "ROOT SCORE: " << root->score << std::endl;+ //std::cout << "ROOT SCORE: " << root->score << std::endl;while(!q.empty() && results.size() < 10) {art_node *n = (art_node *) q.top();@@ -842,18 +823,18 @@ static int topk_iter(art_node *root, std::vector<art_leaf*> & results) {int idx;switch (n->type) {case NODE4:- std::cout << "\nNODE4, SCORE: " << n->score << std::endl;+ //std::cout << "\nNODE4, SCORE: " << n->score << std::endl;for (int i=0; i < n->num_children; i++) {art_node* child = ((art_node4*)n)->children[i];if(IS_LEAF(child)) {art_leaf* l = (art_leaf*)LEAF_RAW(child);- std::cout << "LSCORE: " << l->score << std::endl;+ //std::cout << "LSCORE: " << l->score << std::endl;} else {- std::cout << "SCORE: " << child->score << std::endl;+ //std::cout << "SCORE: " << child->score << std::endl;}q.push(child);}- std::cout << "---" << std::endl;+ //std::cout << "---" << std::endl;break;case NODE16:@@ -881,7 +862,7 @@ static int topk_iter(art_node *root, std::vector<art_leaf*> & results) {break;default:- printf("ABORTING COS OF UNKNOW NODE TYPE: %d\n", n->type);+ printf("ABORTING BECAUSE OF UNKNOWN NODE TYPE: %d\n", n->type);abort();}}@@ -970,19 +951,6 @@ static int leaf_prefix_matches(const art_leaf *n, const unsigned char *prefix, ireturn memcmp(n->key, prefix, prefix_len);}-/**- * Check if a leaf value matches the term within a given threshold- * @return true if the fuzzy match succeeds, false otherwise- */-static int leaf_mismatch(const art_leaf *l, const unsigned char *term, int term_len, int max_cost) {- // cheaper to first check if sequences vary greater than max_cost- if(abs(l->key_len - term_len) > max_cost) return abs(l->key_len - term_len);-- int min_len = min(term_len, l->key_len);- return min_len == 0 ? 0 : levenshtein(l->key, min_len, term, min_len);-}--/*** Iterates through the entries pairs in the map,* invoking a callback for each that matches a given prefix.diff --git a/src/main.cpp b/src/main.cppindex a6bb52d3..3eb4d731 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -70,6 +70,34 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint1}}+void find_documents(art_tree & t, string q) {+ /*+ 1. Split q into tokens+ 2. For each token, look up ids using exact lookup+ a. If a token has no result, try again with edit distance of 1, and then 2+ 3. Intersect the lists to find docs that match all results+ 4. Sort the docs based on some ranking criteria+ */+ vector<string> tokens;+ tokenize(q, tokens, " ", true);++ for(auto token: tokens) {+ int max_cost = 0;+ std::vector<art_leaf*> results;++ do {+ art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), max_cost, results);+ max_cost++;+ } while(results.size() != 0 && max_cost <= 2);++ for(auto leaf: results) {+ for(auto i=0; i<leaf->values->ids.getLength(); i++) {++ }+ }+ }+}+int main() {art_tree t;art_tree_init(&t);@@ -88,7 +116,7 @@ int main() {num++;}- const unsigned char *prefix = (const unsigned char *) "launch";+ const unsigned char *prefix = (const unsigned char *) "l";size_t prefix_len = strlen((const char *) prefix);std::vector<art_leaf*> results;@@ -102,7 +130,7 @@ int main() {cout << "Time taken: " << timeMillis << "us" << endl;for(auto leaf: results) {- std::cout << ">>>>/Key: " << leaf->key << std::endl;+ std::cout << ">>>>/Key: " << leaf->key << " - score: " << leaf->score << std::endl;for(uint32_t i=0; i<leaf->values->ids.getLength(); i++) {std::cout << ", ID: " << leaf->values->ids.at(i) << std::endl;}diff --git a/test/documents.txt b/test/documents.txtindex 38d17dc0..5f79f481 100644--- a/test/documents.txt+++ b/test/documents.txt@@ -12,4 +12,4 @@ What do they exactly look for when searching for extraterrestrial intelligence?What were emergency procedures for failure of launch vehicles with nuclear upper stages? 11Mathematics used for F9R flyback and landing 11What considerations have been made for waste produced during colonisation? 11-Do the propellants ionize in chemical rockets? 11\ No newline at end of file+Do late do the propellants ionize in chemical rockets? 10\ No newline at end of file