Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor trie iteration to be score based.
kishorenc46924be7c756
diff --git a/include/art.h b/include/art.hindex bd887209..54f09718 100644--- a/include/art.h+++ b/include/art.h@@ -44,6 +44,7 @@ typedef struct {uint8_t num_children;uint8_t partial_len;unsigned char partial[MAX_PREFIX_LEN];+ int64_t max_score;} art_node;/**diff --git a/src/art.cpp b/src/art.cppindex c0c61efb..c668094f 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -66,16 +66,20 @@ bool compare_art_node_frequency(const art_node *a, const art_node *b) {}bool compare_art_node_score(const art_node* a, const art_node* b) {- int32_t a_value = 0, b_value = 0;+ int64_t a_value = 0, b_value = 0;if(IS_LEAF(a)) {art_leaf* al = (art_leaf *) LEAF_RAW(a);a_value = al->max_score;+ } else {+ a_value = a->max_score;}if(IS_LEAF(b)) {art_leaf* bl = (art_leaf *) LEAF_RAW(b);b_value = bl->max_score;+ } else {+ b_value = b->max_score;}return a_value > b_value;@@ -112,6 +116,7 @@ static art_node* alloc_node(uint8_t type) {abort();}n->type = type;+ n->max_score = 0;return n;}@@ -469,6 +474,7 @@ static uint32_t longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {static void copy_header(art_node *dest, art_node *src) {dest->num_children = src->num_children;dest->partial_len = src->partial_len;+ dest->max_score = src->max_score;memcpy(dest->partial, src->partial, min(MAX_PREFIX_LEN, src->partial_len));}@@ -476,6 +482,7 @@ static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *(void)ref;n->n.num_children++;n->children[c] = (art_node *) child;+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);}static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *child) {@@ -485,6 +492,7 @@ static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *chn->children[pos] = (art_node *) child;n->keys[c] = pos + 1;n->n.num_children++;+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);} else {art_node256 *new_n = (art_node256*)alloc_node(NODE256);for (int i=0;i<256;i++) {@@ -525,6 +533,7 @@ static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *chn->keys[idx] = c;n->children[idx] = (art_node *) child;n->n.num_children++;+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);} else {art_node48 *new_n = (art_node48*)alloc_node(NODE48);@@ -557,6 +566,7 @@ static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *chiln->keys[idx] = c;n->children[idx] = (art_node *) child;n->n.num_children++;+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);} else {art_node16 *new_n = (art_node16*)alloc_node(NODE16);@@ -653,6 +663,8 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *return NULL;}+ n->max_score = MAX(n->max_score, document->score);+// Check if given node has a prefixif (n->partial_len) {// Determine if the prefixes differ, since we need to split@@ -922,10 +934,19 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rart_node *n = (art_node *) q.top();q.pop();+ /*if (IS_LEAF(n)) {+ art_leaf *l = (art_leaf *) LEAF_RAW(n);+ LOG(INFO) << "Top node (leaf) score: " << l->max_score;+ } else {+ LOG(INFO) << "Top node score: " << n->max_score;+ }*/+if (!n) continue;if (IS_LEAF(n)) {art_leaf *l = (art_leaf *) LEAF_RAW(n);+ //LOG(INFO) << "END LEAF SCORE: " << l->max_score;+if(filter_ids_length == 0) {results.push_back(l);} else {@@ -942,7 +963,7 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rint idx;switch (n->type) {case NODE4:- //LOG(INFO) << "\nNODE4, SCORE: " << n->max_token_count;+ //LOG(INFO) << "NODE4, SCORE: " << n->max_score;for (int i=0; i < n->num_children; i++) {art_node* child = ((art_node4*)n)->children[i];q.push(child);@@ -950,25 +971,24 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_rbreak;case NODE16:- //LOG(INFO) << "\nNODE16, SCORE: " << n->max_token_count;+ //LOG(INFO) << "NODE16, SCORE: " << n->max_score;for (int i=0; i < n->num_children; i++) {q.push(((art_node16*)n)->children[i]);}break;case NODE48:- //LOG(INFO) << "\nNODE48, SCORE: " << n->max_token_count;+ //LOG(INFO) << "NODE48, SCORE: " << n->max_score;for (int i=0; i < 256; i++) {idx = ((art_node48*)n)->keys[i];if (!idx) continue;art_node *child = ((art_node48*)n)->children[idx - 1];- //LOG(INFO) << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child);q.push(child);}break;case NODE256:- //LOG(INFO) << "\nNODE256, SCORE: " << n->max_token_count;+ //LOG(INFO) << "NODE256, SCORE: " << n->max_score;for (int i=0; i < 256; i++) {if (!((art_node256*)n)->children[i]) continue;q.push(((art_node256*)n)->children[i]);diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex 2dc441b0..06762461 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -49,7 +49,7 @@ void benchmark_hn_titles(char* file_path) {collectionManager.init(store, 4, "abcd");collectionManager.load();- Collection *collection = collectionManager.get_collection("hnstories_direct");+ Collection *collection = collectionManager.get_collection("hnstories_direct").get();if(collection == nullptr) {collection = collectionManager.create_collection("hnstories_direct", 4, fields_to_index, "points").get();}@@ -119,7 +119,7 @@ void benchmark_reactjs_pages(char* file_path) {collectionManager.init(store, 4, "abcd");collectionManager.load();- Collection *collection = collectionManager.get_collection("reactjs_pages");+ Collection* collection = collectionManager.get_collection("reactjs_pages").get();if(collection == nullptr) {collection = collectionManager.create_collection("reactjs_pages", 4, fields_to_index, "dummy_sorting_field").get();}@@ -167,12 +167,34 @@ void benchmark_reactjs_pages(char* file_path) {std::cout << "Results total: " << results_total << std::endl;}+void generate_word_freq() {+ std::ifstream infile("/tmp/unigram_freq.jsonl");+ std::ofstream outfile("/tmp/eng_words.jsonl", std::ios_base::app);++ std::string json_line;+ while (std::getline(infile, json_line)) {+ try {+ nlohmann::json obj = nlohmann::json::parse(json_line);+ obj["count"] = uint64_t((double(obj["count"].get<uint64_t>()) / 23135851162) * 1000000000);+ std::string json_str = obj.dump();+ outfile << json_str << std::endl;+ } catch(...) {+ LOG(ERROR) << "Failed parsing: " << json_line;+ }+ }++ infile.close();+ outfile.close();+}+int main(int argc, char* argv[]) {srand(time(NULL));- system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");+// system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");- benchmark_hn_titles(argv[1]);+// benchmark_hn_titles(argv[1]);// benchmark_reactjs_pages(argv[1]);+ generate_word_freq();+return 0;}\ No newline at end of file