Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Append offset related fields to the art leaf during insertion.

Kishore Nallan47df6201b1f1
diff --git a/include/art.h b/include/art.h
index a038552e..4dcd9a43 100644
--- a/include/art.h
+++ b/include/art.h
@@ -93,10 +93,13 @@ typedef struct {
/*
* Represents a document to be indexed.
+ * `offsets` refer to the index locations where a token appeared in the document
*/
typedef struct {
uint16_t score;
uint32_t id;
+ uint32_t offsets_len;
+ uint32_t* offsets;
} art_document;
/**
diff --git a/include/forarray.h b/include/forarray.h
index 0b0b0aa5..3d1cdb7b 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -55,6 +55,12 @@ public:
return for_select(in, index);
}
+ bool contains(uint32_t value) {
+ uint32_t actual;
+ for_lower_bound_search(in, length, value, &actual);
+ return actual == value;
+ }
+
uint32_t getSizeInBytes() {
return size_bytes;
}
diff --git a/src/art.cpp b/src/art.cpp
index 1132f5ca..40a9b98c 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -328,10 +328,20 @@ art_leaf* art_maximum(art_tree *t) {
return maximum((art_node*)t->root);
}
+static void add_document_to_leaf(const art_document *document, const art_leaf *l) {
+ l->values->ids.append_sorted(document->id);
+ uint32_t curr_index = l->values->offsets.getLength();
+ l->values->offsets.append_sorted(document->offsets_len);
+ for(uint32_t i=0; i<document->offsets_len; i++) {
+ l->values->offsets.append_sorted(document->offsets[i]);
+ }
+ l->values->offset_index.append_sorted(curr_index);
+}
+
static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);
l->values = new art_values;
- l->values->ids.append_sorted(document->id);
+ add_document_to_leaf(document, l);
l->score = MAX(l->score, document->score); // somehow std::max does not seem to work here - always returns 0
l->key_len = key_len;
memcpy(l->key, key, key_len);
@@ -518,9 +528,12 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Check if we are updating an existing value
if (!leaf_matches(l, key, key_len, depth)) {
+ // updates are not supported
+ if(l->values->ids.contains(document->id)) return NULL;
+
*old = 1;
art_values *old_val = l->values;
- l->values->ids.append_sorted(document->id);
+ add_document_to_leaf(document, l);
l->score = MAX(l->score, document->score);
return old_val;
}
diff --git a/src/main.cpp b/src/main.cpp
index a9d2e650..fd893a9e 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -8,6 +8,7 @@
#include <numeric>
#include <time.h>
#include <art.h>
+#include <unordered_map>
#include "art.h"
#include "topster.h"
#include "forarray.h"
@@ -63,24 +64,41 @@ void benchmark_heap_array() {
}
void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint16_t score) {
- for(auto & token: tokens){
- std::transform(token.begin(), token.end(), token.begin(), ::tolower);
+ unordered_map<string, vector<uint32_t>> token_to_offsets;
+
+ for(uint32_t i=0; i<tokens.size(); i++) {
+ auto token = tokens[i];
+ if(token_to_offsets.count(token) > 0) {
+ token_to_offsets[token].push_back(i);
+ } else {
+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);
+ token_to_offsets[token] = vector<uint32_t>{i};
+ }
+ }
+
+ for(auto & kv: token_to_offsets) {
art_document document;
document.id = doc_id;
document.score = score;
- art_insert(&t, (const unsigned char *) token.c_str(), token.length(), &document);
+ document.offsets_len = (uint32_t) kv.second.size();
+ document.offsets = new uint32_t[kv.second.size()];
+ for(auto i=0; i<kv.second.size(); i++) {
+ document.offsets[i] = kv.second[i];
+ }
+ art_insert(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length(), &document);
+ delete document.offsets;
}
}
+/*
+ 1. Split q into tokens
+ 2. For each token, look up ids using exact lookup
+ a. If a token has no result, try again with edit distance of 1, and then 2
+ 3. Do a limited cartesian product of the word suggestions for each token to form possible corrected search phrases
+ 4. Intersect the lists to find docs that match each phrase
+ 5. Sort the docs based on some ranking criteria
+ */
void find_documents(art_tree & t, string q) {
- /*
- 1. Split q into tokens
- 2. For each token, look up ids using exact lookup
- a. If a token has no result, try again with edit distance of 1, and then 2
- 3. Do a limited cartesian product of the word suggestions for each token to form possible corrected search phrases
- 4. Intersect the lists to find docs that match each phrase
- 5. Sort the docs based on some ranking criteria
- */
vector<string> tokens;
tokenize(q, tokens, " ", true);
@@ -96,7 +114,7 @@ void find_documents(art_tree & t, string q) {
}
}
- cout << "token_leaves.size(): " << token_leaves.size() << endl;
+ //cout << "token_leaves.size(): " << token_leaves.size() << endl;
const size_t combination_limit = 10;
auto product = []( long long a, vector<art_leaf*>& b ) { return a*b.size(); };