Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Rename `score` to `max_score` for internal node and leaf structs.

Kishore Nallanbb0e7aefb9bd
diff --git a/include/art.h b/include/art.h
index 4dcd9a43..d35e7cc2 100644
--- a/include/art.h
+++ b/include/art.h
@@ -39,9 +39,9 @@ typedef int(*art_callback)(void *data, const unsigned char *key, uint32_t key_le
* of all the various node sizes
*/
typedef struct {
- uint16_t score;
uint8_t type;
uint8_t num_children;
+ uint16_t max_score;
uint32_t partial_len;
unsigned char partial[MAX_PREFIX_LEN];
} art_node;
@@ -107,7 +107,7 @@ typedef struct {
* of arbitrary size, as they include the key.
*/
typedef struct {
- uint16_t score;
+ uint16_t max_score;
uint32_t key_len;
art_values* values;
unsigned char key[];
diff --git a/src/art.cpp b/src/art.cpp
index 03790109..6da5ea17 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -28,7 +28,7 @@
* We order the leaves descending based on the score.
*/
bool compare_art_leaf(const art_leaf* a, const art_leaf* b) {
- return a->score > b->score;
+ return a->max_score > b->max_score;
}
/**
@@ -352,7 +352,7 @@ static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_docum
art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);
l->values = new art_values;
add_document_to_leaf(document, l);
- l->score = MAX(l->score, document->score); // somehow std::max does not seem to work here - always returns 0
+ l->max_score = MAX(l->max_score, document->score); // somehow std::max does not seem to work here - always returns 0
l->key_len = key_len;
memcpy(l->key, key, key_len);
return l;
@@ -369,7 +369,7 @@ static uint32_t longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {
}
static void copy_header(art_node *dest, art_node *src) {
- dest->score = src->score;
+ dest->max_score = src->max_score;
dest->num_children = src->num_children;
dest->partial_len = src->partial_len;
memcpy(dest->partial, src->partial, min(MAX_PREFIX_LEN, src->partial_len));
@@ -377,7 +377,7 @@ static void copy_header(art_node *dest, art_node *src) {
static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child) {
(void)ref;
- n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) child)->max_score);
n->n.num_children++;
n->children[c] = (art_node *) child;
}
@@ -386,7 +386,7 @@ static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *ch
if (n->n.num_children < 48) {
int pos = 0;
while (n->children[pos]) pos++;
- n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) child)->max_score);
n->children[pos] = (art_node *) child;
n->keys[c] = pos + 1;
n->n.num_children++;
@@ -427,7 +427,7 @@ static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *ch
idx = n->n.num_children;
// Set the child
- n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) child)->max_score);
n->keys[idx] = c;
n->children[idx] = (art_node *) child;
n->n.num_children++;
@@ -462,9 +462,9 @@ static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *chil
// Insert element
if (IS_LEAF(child)) {
- n->n.score = MAX(n->n.score, ((art_leaf *) child)->score);
+ n->n.max_score = MAX(n->n.max_score, ((art_leaf *) child)->max_score);
} else {
- n->n.score = MAX(n->n.score, ((art_node *) child)->score);
+ n->n.max_score = MAX(n->n.max_score, ((art_node *) child)->max_score);
}
n->keys[idx] = c;
@@ -544,7 +544,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
*old = 1;
art_values *old_val = l->values;
add_document_to_leaf(document, l);
- l->score = MAX(l->score, document->score);
+ l->max_score = MAX(l->max_score, document->score);
return old_val;
}
@@ -557,7 +557,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Determine longest prefix
uint32_t longest_prefix = longest_common_prefix(l, l2, depth);
new_n->n.partial_len = longest_prefix;
- new_n->n.score = (uint16_t) MAX(l->score, (const uint16_t &) document->score);;
+ new_n->n.max_score = (uint16_t) MAX(l->max_score, (const uint16_t &) document->score);;
memcpy(new_n->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));
// Add the leafs to the new node4
*ref = (art_node*)new_n;
@@ -566,7 +566,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
return NULL;
}
- n->score = (uint16_t) MAX(n->score, (const uint16_t &) document->score);
+ n->max_score = (uint16_t) MAX(n->max_score, (const uint16_t &) document->score);
// Check if given node has a prefix
if (n->partial_len) {
@@ -581,7 +581,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
art_node4 *new_n = (art_node4*)alloc_node(NODE4);
*ref = (art_node*)new_n;
new_n->n.partial_len = prefix_diff;
- new_n->n.score = (uint16_t) MAX(n->score, (const uint16_t &) document->score);
+ new_n->n.max_score = (uint16_t) MAX(n->max_score, (const uint16_t &) document->score);
memcpy(new_n->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff));
// Adjust the prefix of the old node
@@ -817,10 +817,10 @@ static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*>
if(l->key_len == term_len) {
lscore = std::numeric_limits<uint16_t>::max();
} else {
- lscore = l->score;
+ lscore = l->max_score;
}
} else {
- lscore = left->score;
+ lscore = left->max_score;
}
if(IS_LEAF(right)) {
@@ -828,10 +828,10 @@ static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*>
if(r->key_len == term_len) {
rscore = std::numeric_limits<uint16_t>::max();
} else {
- rscore = r->score;
+ rscore = r->max_score;
}
} else {
- rscore = right->score;
+ rscore = right->max_score;
}
return lscore < rscore;
diff --git a/src/main.cpp b/src/main.cpp
index 0871dc73..ea3a9a84 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -77,6 +77,19 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_
document.score = score;
document.offsets_len = (uint32_t) kv.second.size();
document.offsets = new uint32_t[kv.second.size()];
+
+ uint32_t num_docs = 0;
+
+ vector<art_leaf*> results;
+ art_iter_fuzzy_prefix(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length(), 0, 1, results);
+ if(results.size() == 1) {
+ num_docs = results[0]->values->ids.getLength();
+ }
+
+ document.score = (uint16_t) (((uint16_t) num_docs) + 1);
+
+ //cout << "Inserting " << kv.first << " with score: " << document.score << endl;
+
for(auto i=0; i<kv.second.size(); i++) {
document.offsets[i] = kv.second[i];
}
@@ -191,13 +204,23 @@ void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores,
//cin.get();
}
+std::string ReplaceAll(std::string str, const std::string& from, const std::string& to) {
+ size_t start_pos = 0;
+ while((start_pos = str.find(from, start_pos)) != std::string::npos) {
+ str.replace(start_pos, from.length(), to);
+ start_pos += to.length(); // Handles case where 'to' is a substring of 'from'
+ }
+ return str;
+}
+
int main() {
art_tree t;
art_tree_init(&t);
unordered_map<uint32_t, uint16_t> docscores;
- std::ifstream infile("/data/hnstories.tsv");
+ std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");
+ //std::ifstream infile("/data/hnstories.tsv");
std::string line;
uint32_t doc_id = 1;
@@ -205,6 +228,8 @@ int main() {
while (std::getline(infile, line)) {
vector<string> parts;
tokenize(line, parts, "\t", true);
+ line = ReplaceAll(line, "\"", "");
+
vector<string> tokens;
tokenize(parts[0], tokens, " ", true);
@@ -238,7 +263,20 @@ int main() {
std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;
}*/
- find_documents(t, docscores, "the apple", 10);
+// find_documents(t, docscores, "lanch", 10);
+
+ string token = "lanch";
+ vector<art_leaf*> leaves;
+
+ auto begin = std::chrono::high_resolution_clock::now();
+ art_iter_fuzzy_prefix(&t, (const unsigned char *) token.c_str(), (int) token.length(), 1, 4, leaves);
+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ for(auto leaf: leaves) {
+ cout << "Word: " << leaf->key << " - score: " << leaf->max_score << endl;
+ }
+
+ cout << "Time taken: " << timeMillis << "us" << endl;
+
art_tree_destroy(&t);
return 0;
diff --git a/test/documents.txt b/test/documents.txt
index de85437f..6aa52c30 100644
--- a/test/documents.txt
+++ b/test/documents.txt
@@ -10,6 +10,6 @@ How does plant growing medium not scatter around?12Is there research for the opt
Do long term missions receive insurance coverage? 12
What do they exactly look for when searching for extraterrestrial intelligence? 12
What were emergency procedures for failure of launch vehicles with nuclear upper stages? 11
-Mathematics used for F9R flyback and landing 11
-What considerations have been made for waste produced during colonisation? 11
-Do late do the propellants ionize in chemical rockets? 10
\ No newline at end of file
+Mathematics used for F9R flyback lunch and landing 11
+What considerations have been made lunch for waste produced during colonisation? 11
+Do late do the propellants lunch ionize in chemical rockets? 10
\ No newline at end of file