Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor trie iteration to be score based.

kishorenc46924be7c756
diff --git a/include/art.h b/include/art.h
index bd887209..54f09718 100644
--- a/include/art.h
+++ b/include/art.h
@@ -44,6 +44,7 @@ typedef struct {
uint8_t num_children;
uint8_t partial_len;
unsigned char partial[MAX_PREFIX_LEN];
+ int64_t max_score;
} art_node;
/**
diff --git a/src/art.cpp b/src/art.cpp
index c0c61efb..c668094f 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -66,16 +66,20 @@ bool compare_art_node_frequency(const art_node *a, const art_node *b) {
}
bool compare_art_node_score(const art_node* a, const art_node* b) {
- int32_t a_value = 0, b_value = 0;
+ int64_t a_value = 0, b_value = 0;
if(IS_LEAF(a)) {
art_leaf* al = (art_leaf *) LEAF_RAW(a);
a_value = al->max_score;
+ } else {
+ a_value = a->max_score;
}
if(IS_LEAF(b)) {
art_leaf* bl = (art_leaf *) LEAF_RAW(b);
b_value = bl->max_score;
+ } else {
+ b_value = b->max_score;
}
return a_value > b_value;
@@ -112,6 +116,7 @@ static art_node* alloc_node(uint8_t type) {
abort();
}
n->type = type;
+ n->max_score = 0;
return n;
}
@@ -469,6 +474,7 @@ static uint32_t longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {
static void copy_header(art_node *dest, art_node *src) {
dest->num_children = src->num_children;
dest->partial_len = src->partial_len;
+ dest->max_score = src->max_score;
memcpy(dest->partial, src->partial, min(MAX_PREFIX_LEN, src->partial_len));
}
@@ -476,6 +482,7 @@ static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *
(void)ref;
n->n.num_children++;
n->children[c] = (art_node *) child;
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
}
static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *child) {
@@ -485,6 +492,7 @@ static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *ch
n->children[pos] = (art_node *) child;
n->keys[c] = pos + 1;
n->n.num_children++;
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
} else {
art_node256 *new_n = (art_node256*)alloc_node(NODE256);
for (int i=0;i<256;i++) {
@@ -525,6 +533,7 @@ static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *ch
n->keys[idx] = c;
n->children[idx] = (art_node *) child;
n->n.num_children++;
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
} else {
art_node48 *new_n = (art_node48*)alloc_node(NODE48);
@@ -557,6 +566,7 @@ static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *chil
n->keys[idx] = c;
n->children[idx] = (art_node *) child;
n->n.num_children++;
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
} else {
art_node16 *new_n = (art_node16*)alloc_node(NODE16);
@@ -653,6 +663,8 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
return NULL;
}
+ n->max_score = MAX(n->max_score, document->score);
+
// Check if given node has a prefix
if (n->partial_len) {
// Determine if the prefixes differ, since we need to split
@@ -922,10 +934,19 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
art_node *n = (art_node *) q.top();
q.pop();
+ /*if (IS_LEAF(n)) {
+ art_leaf *l = (art_leaf *) LEAF_RAW(n);
+ LOG(INFO) << "Top node (leaf) score: " << l->max_score;
+ } else {
+ LOG(INFO) << "Top node score: " << n->max_score;
+ }*/
+
if (!n) continue;
if (IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
+ //LOG(INFO) << "END LEAF SCORE: " << l->max_score;
+
if(filter_ids_length == 0) {
results.push_back(l);
} else {
@@ -942,7 +963,7 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
int idx;
switch (n->type) {
case NODE4:
- //LOG(INFO) << "\nNODE4, SCORE: " << n->max_token_count;
+ //LOG(INFO) << "NODE4, SCORE: " << n->max_score;
for (int i=0; i < n->num_children; i++) {
art_node* child = ((art_node4*)n)->children[i];
q.push(child);
@@ -950,25 +971,24 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
break;
case NODE16:
- //LOG(INFO) << "\nNODE16, SCORE: " << n->max_token_count;
+ //LOG(INFO) << "NODE16, SCORE: " << n->max_score;
for (int i=0; i < n->num_children; i++) {
q.push(((art_node16*)n)->children[i]);
}
break;
case NODE48:
- //LOG(INFO) << "\nNODE48, SCORE: " << n->max_token_count;
+ //LOG(INFO) << "NODE48, SCORE: " << n->max_score;
for (int i=0; i < 256; i++) {
idx = ((art_node48*)n)->keys[i];
if (!idx) continue;
art_node *child = ((art_node48*)n)->children[idx - 1];
- //LOG(INFO) << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child);
q.push(child);
}
break;
case NODE256:
- //LOG(INFO) << "\nNODE256, SCORE: " << n->max_token_count;
+ //LOG(INFO) << "NODE256, SCORE: " << n->max_score;
for (int i=0; i < 256; i++) {
if (!((art_node256*)n)->children[i]) continue;
q.push(((art_node256*)n)->children[i]);
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index 2dc441b0..06762461 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -49,7 +49,7 @@ void benchmark_hn_titles(char* file_path) {
collectionManager.init(store, 4, "abcd");
collectionManager.load();
- Collection *collection = collectionManager.get_collection("hnstories_direct");
+ Collection *collection = collectionManager.get_collection("hnstories_direct").get();
if(collection == nullptr) {
collection = collectionManager.create_collection("hnstories_direct", 4, fields_to_index, "points").get();
}
@@ -119,7 +119,7 @@ void benchmark_reactjs_pages(char* file_path) {
collectionManager.init(store, 4, "abcd");
collectionManager.load();
- Collection *collection = collectionManager.get_collection("reactjs_pages");
+ Collection* collection = collectionManager.get_collection("reactjs_pages").get();
if(collection == nullptr) {
collection = collectionManager.create_collection("reactjs_pages", 4, fields_to_index, "dummy_sorting_field").get();
}
@@ -167,12 +167,34 @@ void benchmark_reactjs_pages(char* file_path) {
std::cout << "Results total: " << results_total << std::endl;
}
+void generate_word_freq() {
+ std::ifstream infile("/tmp/unigram_freq.jsonl");
+ std::ofstream outfile("/tmp/eng_words.jsonl", std::ios_base::app);
+
+ std::string json_line;
+ while (std::getline(infile, json_line)) {
+ try {
+ nlohmann::json obj = nlohmann::json::parse(json_line);
+ obj["count"] = uint64_t((double(obj["count"].get<uint64_t>()) / 23135851162) * 1000000000);
+ std::string json_str = obj.dump();
+ outfile << json_str << std::endl;
+ } catch(...) {
+ LOG(ERROR) << "Failed parsing: " << json_line;
+ }
+ }
+
+ infile.close();
+ outfile.close();
+}
+
int main(int argc, char* argv[]) {
srand(time(NULL));
- system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");
+// system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");
- benchmark_hn_titles(argv[1]);
+// benchmark_hn_titles(argv[1]);
// benchmark_reactjs_pages(argv[1]);
+ generate_word_freq();
+
return 0;
}
\ No newline at end of file