Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Append offset related fields to the art leaf during insertion.
Kishore Nallan47df6201b1f1
diff --git a/include/art.h b/include/art.hindex a038552e..4dcd9a43 100644--- a/include/art.h+++ b/include/art.h@@ -93,10 +93,13 @@ typedef struct {/** Represents a document to be indexed.+ * `offsets` refer to the index locations where a token appeared in the document*/typedef struct {uint16_t score;uint32_t id;+ uint32_t offsets_len;+ uint32_t* offsets;} art_document;/**diff --git a/include/forarray.h b/include/forarray.hindex 0b0b0aa5..3d1cdb7b 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -55,6 +55,12 @@ public:return for_select(in, index);}+ bool contains(uint32_t value) {+ uint32_t actual;+ for_lower_bound_search(in, length, value, &actual);+ return actual == value;+ }+uint32_t getSizeInBytes() {return size_bytes;}diff --git a/src/art.cpp b/src/art.cppindex 1132f5ca..40a9b98c 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -328,10 +328,20 @@ art_leaf* art_maximum(art_tree *t) {return maximum((art_node*)t->root);}+static void add_document_to_leaf(const art_document *document, const art_leaf *l) {+ l->values->ids.append_sorted(document->id);+ uint32_t curr_index = l->values->offsets.getLength();+ l->values->offsets.append_sorted(document->offsets_len);+ for(uint32_t i=0; i<document->offsets_len; i++) {+ l->values->offsets.append_sorted(document->offsets[i]);+ }+ l->values->offset_index.append_sorted(curr_index);+}+static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);l->values = new art_values;- l->values->ids.append_sorted(document->id);+ add_document_to_leaf(document, l);l->score = MAX(l->score, document->score); // somehow std::max does not seem to work here - always returns 0l->key_len = key_len;memcpy(l->key, key, key_len);@@ -518,9 +528,12 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Check if we are updating an existing valueif (!leaf_matches(l, key, key_len, depth)) {+ // updates are not supported+ if(l->values->ids.contains(document->id)) return NULL;+*old = 1;art_values *old_val = l->values;- l->values->ids.append_sorted(document->id);+ add_document_to_leaf(document, l);l->score = MAX(l->score, document->score);return old_val;}diff --git a/src/main.cpp b/src/main.cppindex a9d2e650..fd893a9e 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -8,6 +8,7 @@#include <numeric>#include <time.h>#include <art.h>+#include <unordered_map>#include "art.h"#include "topster.h"#include "forarray.h"@@ -63,24 +64,41 @@ void benchmark_heap_array() {}void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint16_t score) {- for(auto & token: tokens){- std::transform(token.begin(), token.end(), token.begin(), ::tolower);+ unordered_map<string, vector<uint32_t>> token_to_offsets;++ for(uint32_t i=0; i<tokens.size(); i++) {+ auto token = tokens[i];+ if(token_to_offsets.count(token) > 0) {+ token_to_offsets[token].push_back(i);+ } else {+ std::transform(token.begin(), token.end(), token.begin(), ::tolower);+ token_to_offsets[token] = vector<uint32_t>{i};+ }+ }++ for(auto & kv: token_to_offsets) {art_document document;document.id = doc_id;document.score = score;- art_insert(&t, (const unsigned char *) token.c_str(), token.length(), &document);+ document.offsets_len = (uint32_t) kv.second.size();+ document.offsets = new uint32_t[kv.second.size()];+ for(auto i=0; i<kv.second.size(); i++) {+ document.offsets[i] = kv.second[i];+ }+ art_insert(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length(), &document);+ delete document.offsets;}}+/*+ 1. Split q into tokens+ 2. For each token, look up ids using exact lookup+ a. If a token has no result, try again with edit distance of 1, and then 2+ 3. Do a limited cartesian product of the word suggestions for each token to form possible corrected search phrases+ 4. Intersect the lists to find docs that match each phrase+ 5. Sort the docs based on some ranking criteria+ */void find_documents(art_tree & t, string q) {- /*- 1. Split q into tokens- 2. For each token, look up ids using exact lookup- a. If a token has no result, try again with edit distance of 1, and then 2- 3. Do a limited cartesian product of the word suggestions for each token to form possible corrected search phrases- 4. Intersect the lists to find docs that match each phrase- 5. Sort the docs based on some ranking criteria- */vector<string> tokens;tokenize(q, tokens, " ", true);@@ -96,7 +114,7 @@ void find_documents(art_tree & t, string q) {}}- cout << "token_leaves.size(): " << token_leaves.size() << endl;+ //cout << "token_leaves.size(): " << token_leaves.size() << endl;const size_t combination_limit = 10;auto product = []( long long a, vector<art_leaf*>& b ) { return a*b.size(); };