Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Removed redundant token_count field from the leaf.

Kishore Nallan1c09ec38a895
diff --git a/TODO.md b/TODO.md
index 643b1506..7cb94a95 100644
--- a/TODO.md
+++ b/TODO.md
@@ -28,5 +28,5 @@
**Refactoring**
-- `token_count` in leaf is redundant: can be accessed from value
-- ~~storing length in `offsets` is redundant: it can be found by looking up value of the next index in `offset_index`~~
+- ~~`token_count` in leaf is redundant: can be accessed from value~~
+- ~~storing length in `offsets` is redundant: it can be found by looking up value of the next index in offset_index~~
\ No newline at end of file
diff --git a/include/art.h b/include/art.h
index 14c2b0f5..97ecfd35 100644
--- a/include/art.h
+++ b/include/art.h
@@ -110,7 +110,6 @@ typedef struct {
typedef struct {
art_values* values;
uint16_t max_score;
- uint32_t token_count;
uint32_t key_len;
unsigned char key[];
} art_leaf;
diff --git a/src/art.cpp b/src/art.cpp
index 9e8f01cc..b0877047 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -28,7 +28,7 @@
* We order the leaves descending based on the score.
*/
bool compare_art_leaf(const art_leaf* a, const art_leaf* b) {
- return a->token_count > b->token_count;
+ return a->values->ids.getLength() > b->values->ids.getLength();
}
bool compare_art_node(const art_node* a, const art_node* b) {
@@ -36,14 +36,14 @@ bool compare_art_node(const art_node* a, const art_node* b) {
if(IS_LEAF(a)) {
art_leaf* al = (art_leaf *) LEAF_RAW(a);
- a_score = al->token_count;
+ a_score = al->values->ids.getLength();
} else {
a_score = a->max_token_count;
}
if(IS_LEAF(b)) {
art_leaf* bl = (art_leaf *) LEAF_RAW(b);
- b_score = bl->token_count;
+ b_score = bl->values->ids.getLength();
} else {
b_score = b->max_token_count;
}
@@ -361,7 +361,6 @@ art_leaf* art_maximum(art_tree *t) {
static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {
leaf->max_score = MAX(leaf->max_score, document->score);
- leaf->token_count += document->offsets_len;
leaf->values->ids.append_sorted(document->id);
uint32_t curr_index = leaf->values->offsets.getLength();
leaf->values->offset_index.append_sorted(curr_index);
@@ -374,7 +373,6 @@ static void add_document_to_leaf(const art_document *document, art_leaf *leaf) {
static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);
l->values = new art_values;
- l->token_count = 0;
l->max_score = 0;
l->key_len = key_len;
memcpy(l->key, key, key_len);
@@ -403,7 +401,7 @@ static void copy_header(art_node *dest, art_node *src) {
static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child) {
(void)ref;
n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
- n->n.max_token_count = MAX(n->n.max_token_count, ((art_leaf *) LEAF_RAW(child))->token_count);
+ n->n.max_token_count = MAX(n->n.max_token_count, ((art_leaf *) LEAF_RAW(child))->values->ids.getLength());
n->n.num_children++;
n->children[c] = (art_node *) child;
}
@@ -413,7 +411,7 @@ static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *ch
int pos = 0;
while (n->children[pos]) pos++;
n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
- n->n.max_token_count = MAX(n->n.max_token_count, ((art_leaf *) LEAF_RAW(child))->token_count);
+ n->n.max_token_count = MAX(n->n.max_token_count, ((art_leaf *) LEAF_RAW(child))->values->ids.getLength());
n->children[pos] = (art_node *) child;
n->keys[c] = pos + 1;
n->n.num_children++;
@@ -455,7 +453,7 @@ static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *ch
// Set the child
n->n.max_score = MAX(n->n.max_score, ((art_leaf *) LEAF_RAW(child))->max_score);
- n->n.max_token_count = MAX(n->n.max_token_count, ((art_leaf *) LEAF_RAW(child))->token_count);
+ n->n.max_token_count = MAX(n->n.max_token_count, ((art_leaf *) LEAF_RAW(child))->values->ids.getLength());
n->keys[idx] = c;
n->children[idx] = (art_node *) child;
n->n.num_children++;
@@ -489,7 +487,7 @@ static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *chil
(n->n.num_children - idx)*sizeof(void*));
uint16_t child_max_score = IS_LEAF(child) ? ((art_leaf *) LEAF_RAW(child))->max_score : ((art_node *) child)->max_score;
- uint32_t child_token_count = IS_LEAF(child) ? ((art_leaf *) LEAF_RAW(child))->token_count : ((art_node *) child)->max_token_count;
+ uint32_t child_token_count = IS_LEAF(child) ? ((art_leaf *) LEAF_RAW(child))->values->ids.getLength() : ((art_node *) child)->max_token_count;
n->n.max_score = MAX(n->n.max_score, child_max_score);
n->n.max_token_count = MAX(n->n.max_token_count, child_token_count);
@@ -835,7 +833,7 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {
static uint32_t get_score(art_node* child) {
if (IS_LEAF(child)) {
art_leaf *l = (art_leaf *) LEAF_RAW(child);
- return l->token_count;
+ return l->values->ids.getLength();
}
return child->max_token_count;
@@ -848,14 +846,14 @@ static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*>
int lscore, rscore;
if (IS_LEAF(left)) {
art_leaf *l = (art_leaf *) LEAF_RAW(left);
- lscore = l->token_count;
+ lscore = l->values->ids.getLength();
} else {
lscore = left->max_token_count;
}
if(IS_LEAF(right)) {
art_leaf *r = (art_leaf *) LEAF_RAW(right);
- rscore = r->token_count;
+ rscore = r->values->ids.getLength();
} else {
rscore = right->max_token_count;
}
diff --git a/src/main.cpp b/src/main.cpp
index a4bf1342..3eacb753 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -11,7 +11,7 @@
using namespace std;
int main() {
- Collection *index = new Collection();
+ Collection *collection = new Collection();
std::ifstream infile("/Users/kishore/others/wreally/typesense/test/documents.txt");
//std::ifstream infile("/Users/kishore/Downloads/hnstories.tsv");
@@ -27,15 +27,15 @@ int main() {
StringUtils::tokenize(parts[0], tokens, " ", true);
if(parts.size() != 2) continue;
- index->add(tokens, stoi(parts[1]));
+ collection->add(tokens, stoi(parts[1]));
}
cout << "FINISHED INDEXING!" << endl << flush;
auto begin = std::chrono::high_resolution_clock::now();
- index->search("plant", 100);
+ collection->search("plant", 100);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
cout << "Time taken: " << timeMillis << "us" << endl;
- delete index;
+ delete collection;
return 0;
}
\ No newline at end of file