Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Using a proper document struct when representing leaf values.
Kishore Nallancb3b0e1a6e51
diff --git a/.gitmodules b/.gitmodulesindex 183e68af..216c807c 100644--- a/.gitmodules+++ b/.gitmodules@@ -1,12 +1,4 @@-[submodule "external/croaring"]- path = external/croaring- url = [email protected]:RoaringBitmap/CRoaring.git- ignore = dirty[submodule "external/for"]path = external/forurl = [email protected]:cruppstahl/for.git- ignore = dirty-[submodule "external/EWAHBoolArray"]- path = external/EWAHBoolArray- url = [email protected]:lemire/EWAHBoolArray.git- ignore = dirty+ ignore = dirty\ No newline at end of filediff --git a/CMakeLists.txt b/CMakeLists.txtindex 297bdb2c..faffab03 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -4,40 +4,8 @@ project(search)set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O2")include_directories(include)-include_directories(external/EWAHBoolArray/headers)-# Setup roaring as a static lib ----include_directories(external/croaring/include external/croaring/include/containers external/croaring/include/misc)+include_directories(external/for)-add_library(array STATIC external/croaring/array.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(array PROPERTIES LINKER_LANGUAGE C)--add_library(roaring_array STATIC external/croaring/roaring_array.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring_array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(roaring_array PROPERTIES LINKER_LANGUAGE C)--add_library(bitset STATIC external/croaring/bitset.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/bitset.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(bitset PROPERTIES LINKER_LANGUAGE C)--add_library(containers STATIC external/croaring/containers.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/containers.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(containers PROPERTIES LINKER_LANGUAGE C)--add_library(run STATIC external/croaring/run.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/run.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(run PROPERTIES LINKER_LANGUAGE C)--add_library(util STATIC external/croaring/util.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/util.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(util PROPERTIES LINKER_LANGUAGE C)--add_library(roaring STATIC external/croaring/roaring.o)-SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)-SET_TARGET_PROPERTIES(roaring PROPERTIES LINKER_LANGUAGE C)--# End Roaring as static lib -----add_executable(search src/art.cpp src/main.cpp)-target_link_libraries(search roaring roaring_array array bitset containers run util for)+add_executable(search external/for/libfor.a src/art.cpp src/main.cpp)+target_link_libraries(search for)diff --git a/external/EWAHBoolArray b/external/EWAHBoolArraydeleted file mode 160000index 80881379..00000000--- a/external/EWAHBoolArray+++ /dev/null@@ -1 +0,0 @@-Subproject commit 80881379f8a582f45dda1be9edfc84d244846427diff --git a/external/croaring b/external/croaringdeleted file mode 160000index 230b6138..00000000--- a/external/croaring+++ /dev/null@@ -1 +0,0 @@-Subproject commit 230b613864fa970b0b4ce7f6aedfc2ee6dbaab1bdiff --git a/include/art.h b/include/art.hindex 43b69277..3f9186d4 100644--- a/include/art.h+++ b/include/art.h@@ -3,6 +3,7 @@#include <stdint.h>#include <stdbool.h>+#include "forarray.h"#define IGNORE_PRINTF 1@@ -78,6 +79,23 @@ typedef struct {art_node *children[256];} art_node256;+/**+ * Container for holding the documents that belong to a leaf.+ */+typedef struct {+ forarray ids;+ forarray offset_index;+ forarray offsets;+} art_values;++/*+ * Represents a document to be indexed.+ */+typedef struct {+ uint16_t score;+ uint32_t id;+} art_document;+/*** Represents a leaf. These are* of arbitrary size, as they include the key.@@ -85,7 +103,7 @@ typedef struct {typedef struct {uint16_t score;uint32_t key_len;- void *value;+ art_values* values;unsigned char key[];} art_leaf;@@ -143,7 +161,7 @@ inline uint64_t art_size(art_tree *t) {* @return NULL if the item was newly inserted, otherwise* the old value pointer is returned.*/-void* art_insert(art_tree *t, const unsigned char *key, int key_len, int score, void *value);+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document);/*** Deletes a value from the ART treediff --git a/include/forarray.h b/include/forarray.hnew file mode 100644index 00000000..95474651--- /dev/null+++ b/include/forarray.h@@ -0,0 +1,54 @@+#pragma once++#include <stdio.h>+#include <cstdlib>+#include <for.h>++#define FOR_GROWTH_FACTOR 1.3+#define FOR_ELE_SIZE sizeof(uint32_t)+#define FOR_LOAD_FACTOR 0.90++class forarray {+private:+ uint32_t size_bytes = 0;+ uint32_t length_bytes = 0;+ uint32_t length = 0;+ uint8_t* in;+public:+ forarray(const uint32_t n=2) {+ size_bytes = n * FOR_ELE_SIZE;+ in = new uint8_t[size_bytes];+ }++ // returns false if malloc fails+ bool append_sorted(uint32_t value) {+ if(length_bytes > size_bytes*FOR_LOAD_FACTOR) {+ // grow the array first+ size_t new_size = (size_t) (size_bytes * FOR_GROWTH_FACTOR);+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);+ if(new_location == NULL) return false;+ in = new_location;+ size_bytes = (uint32_t) new_size;+ }++ uint32_t new_length_bytes = for_append_sorted(in, length, value);+ //printf("new_length_bytes: %d, size_bytes: %d\n", new_length_bytes, size_bytes);+ if(new_length_bytes == 0) return false;++ length_bytes = new_length_bytes;+ length++;+ return true;+ }++ uint32_t at(uint32_t index) {+ return for_select(in, index);+ }++ uint32_t getSizeInBytes() {+ return size_bytes;+ }++ void print_stats() {+ printf("length: %d, length_bytes: %d, size_bytes: %d\n", length, length_bytes, size_bytes);+ }+};\ No newline at end of filediff --git a/src/art.cpp b/src/art.cppindex 0815b32d..bc7a6a01 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -260,7 +260,7 @@ void* art_search(const art_tree *t, const unsigned char *key, int key_len) {n = (art_node *) LEAF_RAW(n);// Check if the expanded path matchesif (!leaf_matches((art_leaf*)n, key, key_len, depth)) {- return ((art_leaf*)n)->value;+ return ((art_leaf*)n)->values;}return NULL;}@@ -347,18 +347,19 @@ art_leaf* art_maximum(art_tree *t) {return maximum((art_node*)t->root);}-static art_leaf* make_leaf(const unsigned char *key, int key_len, int score, void *value) {+static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);- l->value = value;- l->score = (uint16_t) score;+ l->values = new art_values;+ l->values->ids.append_sorted(document->id);+ l->score = std::max(l->score, document->score);l->key_len = key_len;memcpy(l->key, key, key_len);return l;}-static int longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {+static uint32_t longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {int max_cmp = min(l1->key_len, l2->key_len) - depth;- int idx;+ uint32_t idx;for (idx=0; idx < max_cmp; idx++) {if (l1->key[depth+idx] != l2->key[depth+idx])return idx;@@ -523,10 +524,10 @@ static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_return idx;}-static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, int key_len, int score, void *value, int depth, int *old) {+static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len, art_document *document, int depth, int *old) {// If we are at a NULL node, inject a leafif (!n) {- *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, score, value));+ *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, document));return NULL;}@@ -537,9 +538,9 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Check if we are updating an existing valueif (!leaf_matches(l, key, key_len, depth)) {*old = 1;- void *old_val = l->value;- l->value = value;- l->score = (uint16_t) score;+ art_values *old_val = l->values;+ l->values->ids.append_sorted(document->id);+ l->score = std::max(l->score, document->score);return old_val;}@@ -547,12 +548,12 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *art_node4 *new_n = (art_node4*)alloc_node(NODE4);// Create a new leaf- art_leaf *l2 = make_leaf(key, key_len, score, value);+ art_leaf *l2 = make_leaf(key, key_len, document);// Determine longest prefix- int longest_prefix = longest_common_prefix(l, l2, depth);+ uint32_t longest_prefix = longest_common_prefix(l, l2, depth);new_n->n.partial_len = longest_prefix;- new_n->n.score = (uint16_t) std::max(l->score, (const uint16_t &) score);;+ new_n->n.score = (uint16_t) std::max(l->score, (const uint16_t &) document->score);;memcpy(new_n->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));// Add the leafs to the new node4*ref = (art_node*)new_n;@@ -561,7 +562,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *return NULL;}- n->score = (uint16_t) std::max(n->score, (const uint16_t &) score);+ n->score = (uint16_t) std::max(n->score, (const uint16_t &) document->score);// Check if given node has a prefixif (n->partial_len) {@@ -576,7 +577,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *art_node4 *new_n = (art_node4*)alloc_node(NODE4);*ref = (art_node*)new_n;new_n->n.partial_len = prefix_diff;- new_n->n.score = (uint16_t) std::max(n->score, (const uint16_t &) score);+ new_n->n.score = (uint16_t) std::max(n->score, (const uint16_t &) document->score);memcpy(new_n->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff));// Adjust the prefix of the old node@@ -594,7 +595,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *}// Insert the new leaf- art_leaf *l = make_leaf(key, key_len, score, value);+ art_leaf *l = make_leaf(key, key_len, document);add_child4(new_n, ref, key[depth+prefix_diff], SET_LEAF(l));return NULL;}@@ -604,11 +605,11 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *// Find a child to recurse toart_node **child = find_child(n, key[depth]);if (child) {- return recursive_insert(*child, child, key, key_len, score, value, depth+1, old);+ return recursive_insert(*child, child, key, key_len, document, depth+1, old);}// No child, node goes within us- art_leaf *l = make_leaf(key, key_len, score, value);+ art_leaf *l = make_leaf(key, key_len, document);add_child(n, ref, key[depth], SET_LEAF(l));return NULL;}@@ -622,9 +623,10 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char ** @return NULL if the item was newly inserted, otherwise* the old value pointer is returned.*/-void* art_insert(art_tree *t, const unsigned char *key, int key_len, int score, void *value) {+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document) {int old_val = 0;- void *old = recursive_insert(t->root, &t->root, key, key_len, score, value, 0, &old_val);++ void *old = recursive_insert(t->root, &t->root, key, key_len, document, 0, &old_val);if (!old_val) t->size++;return old;}@@ -792,7 +794,7 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {art_leaf *l = recursive_delete(t->root, &t->root, key, key_len, 0);if (l) {t->size--;- void *old = l->value;+ void *old = l->values;free(l);return old;}@@ -822,7 +824,7 @@ static int topk_iter(art_node *root) {return lscore < rscore;};std::priority_queue<art_node *, std::vector<art_node *>, decltype(cmp)> q(cmp);- std::vector<unsigned char*> results;+ std::vector<art_leaf*> results;q.push(root);std::cout << "ROOT SCORE: " << root->score << std::endl;@@ -834,7 +836,7 @@ static int topk_iter(art_node *root) {if (!n) continue;if (IS_LEAF(n)) {art_leaf *l = (art_leaf *) LEAF_RAW(n);- results.push_back(l->key);+ results.push_back(l);continue;}@@ -887,9 +889,9 @@ static int topk_iter(art_node *root) {printf("OUTSIDE topk_iter: results size: %d\n", results.size());- for(auto res: results) {- std::cout << ">>>>!Key: " << res;- std::cout << ", Value: " << std::endl;+ for(auto leaf: results) {+ std::cout << ">>>>!Key: " << leaf->key;+ std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;}return 0;@@ -902,7 +904,7 @@ static int recursive_iter(art_node *n, art_callback cb, void *data) {if (IS_LEAF(n)) {art_leaf *l = (art_leaf *) LEAF_RAW(n);//printf("REC LEAF len: %d, key: %s\n", l->key_len, l->key);- return cb(data, (const unsigned char*)l->key, l->key_len, l->value);+ return cb(data, (const unsigned char*)l->key, l->key_len, l->values);}//printf("INTERNAL LEAF children: %d, partial_len: %d, partial: %s\n", n->num_children, n->partial_len, n->partial);@@ -1015,7 +1017,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call// Check if the expanded path matchesif (!leaf_prefix_matches((art_leaf*)n, key, key_len)) {art_leaf *l = (art_leaf*)n;- return cb(data, (const unsigned char*)l->key, l->key_len, l->value);+ return cb(data, (const unsigned char*)l->key, l->key_len, l->values);}return 0;}@@ -1118,8 +1120,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,if (IS_LEAF(n)) {printf("IS_LEAF\n");- n = (art_node *) LEAF_RAW(n);- art_leaf *l = (art_leaf *) n;+ art_leaf *l = (art_leaf *) LEAF_RAW(n);int row_min = 0;int current_row[term_len+1];@@ -1144,7 +1145,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,}if(current_row[term_len] <= max_cost) {- return cb(data, (const unsigned char*)l->key, l->key_len, l->value);+ topk_iter(n);}return 0;diff --git a/src/main.cpp b/src/main.cppindex 53b260d2..7763b26d 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -5,17 +5,18 @@#include <chrono>#include <vector>#include <time.h>-#include <ewah.h>+#include <art.h>#include "art.h"#include "topster.h"+#include "forarray.h"#include "util.h"using namespace std;static int test_prefix_cb(void *data, const unsigned char *k, uint32_t k_len, void *val) {- cout << ">>>>Key: ";+ cout << "#>>>>Key: ";printf("%.*s", k_len, k);- cout << ", Value: " << (uintptr_t) val << endl;+ cout << ", ID: " << ((art_values*)val)->ids.at(0) << endl;return 0;}@@ -55,36 +56,13 @@ void benchmark_heap_array() {cout << "Time taken: " << timeMillis << endl;}-void ewahIntersect() {- EWAHBoolArray<uint32_t> b1;- EWAHBoolArray<uint32_t> b2;- EWAHBoolArray<uint32_t> b3;-- for (uint32_t i=0; i < 1000000; ++i) {- b1.set(2*i);- b2.set(3*i);- }-- auto begin1 = std::chrono::high_resolution_clock::now();-- b1.logicaland(b2, b3);-- vector<size_t> arr = b2.toArray();-- long long int timeMillis1 = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin1).count();-- int32_t counter = 0;- for(int i=0; i<arr.size(); i++) {- counter++;- }-- cout << endl << "TIME: " << timeMillis1 << " and counter is: " << counter << endl;-}--void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint32_t score) {+void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint16_t score) {for(auto & token: tokens){std::transform(token.begin(), token.end(), token.begin(), ::tolower);- art_insert(&t, (const unsigned char *) token.c_str(), token.length(), score, (void*)doc_id);+ art_document document;+ document.id = doc_id;+ document.score = score;+ art_insert(&t, (const unsigned char *) token.c_str(), token.length(), &document);}}@@ -106,11 +84,11 @@ int main() {num++;}- const unsigned char *prefix = (const unsigned char *) "wherre";+ const unsigned char *prefix = (const unsigned char *) "propellants";size_t prefix_len = strlen((const char *) prefix);auto begin = std::chrono::high_resolution_clock::now();- art_iter_fuzzy_prefix(&t, prefix, prefix_len, 1, test_prefix_cb, NULL);+ art_iter_fuzzy_prefix(&t, prefix, prefix_len, 2, test_prefix_cb, NULL);long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();// art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);