Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Using a proper document struct when representing leaf values.

Kishore Nallancb3b0e1a6e51
diff --git a/.gitmodules b/.gitmodules
index 183e68af..216c807c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,12 +1,4 @@
-[submodule "external/croaring"]
- path = external/croaring
- url = [email protected]:RoaringBitmap/CRoaring.git
- ignore = dirty
[submodule "external/for"]
path = external/for
url = [email protected]:cruppstahl/for.git
- ignore = dirty
-[submodule "external/EWAHBoolArray"]
- path = external/EWAHBoolArray
- url = [email protected]:lemire/EWAHBoolArray.git
- ignore = dirty
+ ignore = dirty
\ No newline at end of file
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 297bdb2c..faffab03 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,40 +4,8 @@ project(search)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++ -std=gnu++11 -O2")
include_directories(include)
-include_directories(external/EWAHBoolArray/headers)
-# Setup roaring as a static lib ---
-include_directories(external/croaring/include external/croaring/include/containers external/croaring/include/misc)
+include_directories(external/for)
-add_library(array STATIC external/croaring/array.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(array PROPERTIES LINKER_LANGUAGE C)
-
-add_library(roaring_array STATIC external/croaring/roaring_array.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring_array.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(roaring_array PROPERTIES LINKER_LANGUAGE C)
-
-add_library(bitset STATIC external/croaring/bitset.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/bitset.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(bitset PROPERTIES LINKER_LANGUAGE C)
-
-add_library(containers STATIC external/croaring/containers.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/containers.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(containers PROPERTIES LINKER_LANGUAGE C)
-
-add_library(run STATIC external/croaring/run.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/run.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(run PROPERTIES LINKER_LANGUAGE C)
-
-add_library(util STATIC external/croaring/util.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/util.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(util PROPERTIES LINKER_LANGUAGE C)
-
-add_library(roaring STATIC external/croaring/roaring.o)
-SET_SOURCE_FILES_PROPERTIES(external/croaring/roaring.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
-SET_TARGET_PROPERTIES(roaring PROPERTIES LINKER_LANGUAGE C)
-
-# End Roaring as static lib ---
-
-add_executable(search src/art.cpp src/main.cpp)
-target_link_libraries(search roaring roaring_array array bitset containers run util for)
+add_executable(search external/for/libfor.a src/art.cpp src/main.cpp)
+target_link_libraries(search for)
diff --git a/external/EWAHBoolArray b/external/EWAHBoolArray
deleted file mode 160000
index 80881379..00000000
--- a/external/EWAHBoolArray
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 80881379f8a582f45dda1be9edfc84d244846427
diff --git a/external/croaring b/external/croaring
deleted file mode 160000
index 230b6138..00000000
--- a/external/croaring
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 230b613864fa970b0b4ce7f6aedfc2ee6dbaab1b
diff --git a/include/art.h b/include/art.h
index 43b69277..3f9186d4 100644
--- a/include/art.h
+++ b/include/art.h
@@ -3,6 +3,7 @@
#include <stdint.h>
#include <stdbool.h>
+#include "forarray.h"
#define IGNORE_PRINTF 1
@@ -78,6 +79,23 @@ typedef struct {
art_node *children[256];
} art_node256;
+/**
+ * Container for holding the documents that belong to a leaf.
+ */
+typedef struct {
+ forarray ids;
+ forarray offset_index;
+ forarray offsets;
+} art_values;
+
+/*
+ * Represents a document to be indexed.
+ */
+typedef struct {
+ uint16_t score;
+ uint32_t id;
+} art_document;
+
/**
* Represents a leaf. These are
* of arbitrary size, as they include the key.
@@ -85,7 +103,7 @@ typedef struct {
typedef struct {
uint16_t score;
uint32_t key_len;
- void *value;
+ art_values* values;
unsigned char key[];
} art_leaf;
@@ -143,7 +161,7 @@ inline uint64_t art_size(art_tree *t) {
* @return NULL if the item was newly inserted, otherwise
* the old value pointer is returned.
*/
-void* art_insert(art_tree *t, const unsigned char *key, int key_len, int score, void *value);
+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document);
/**
* Deletes a value from the ART tree
diff --git a/include/forarray.h b/include/forarray.h
new file mode 100644
index 00000000..95474651
--- /dev/null
+++ b/include/forarray.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include <stdio.h>
+#include <cstdlib>
+#include <for.h>
+
+#define FOR_GROWTH_FACTOR 1.3
+#define FOR_ELE_SIZE sizeof(uint32_t)
+#define FOR_LOAD_FACTOR 0.90
+
+class forarray {
+private:
+ uint32_t size_bytes = 0;
+ uint32_t length_bytes = 0;
+ uint32_t length = 0;
+ uint8_t* in;
+public:
+ forarray(const uint32_t n=2) {
+ size_bytes = n * FOR_ELE_SIZE;
+ in = new uint8_t[size_bytes];
+ }
+
+ // returns false if malloc fails
+ bool append_sorted(uint32_t value) {
+ if(length_bytes > size_bytes*FOR_LOAD_FACTOR) {
+ // grow the array first
+ size_t new_size = (size_t) (size_bytes * FOR_GROWTH_FACTOR);
+ uint8_t *new_location = (uint8_t *) realloc(in, new_size);
+ if(new_location == NULL) return false;
+ in = new_location;
+ size_bytes = (uint32_t) new_size;
+ }
+
+ uint32_t new_length_bytes = for_append_sorted(in, length, value);
+ //printf("new_length_bytes: %d, size_bytes: %d\n", new_length_bytes, size_bytes);
+ if(new_length_bytes == 0) return false;
+
+ length_bytes = new_length_bytes;
+ length++;
+ return true;
+ }
+
+ uint32_t at(uint32_t index) {
+ return for_select(in, index);
+ }
+
+ uint32_t getSizeInBytes() {
+ return size_bytes;
+ }
+
+ void print_stats() {
+ printf("length: %d, length_bytes: %d, size_bytes: %d\n", length, length_bytes, size_bytes);
+ }
+};
\ No newline at end of file
diff --git a/src/art.cpp b/src/art.cpp
index 0815b32d..bc7a6a01 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -260,7 +260,7 @@ void* art_search(const art_tree *t, const unsigned char *key, int key_len) {
n = (art_node *) LEAF_RAW(n);
// Check if the expanded path matches
if (!leaf_matches((art_leaf*)n, key, key_len, depth)) {
- return ((art_leaf*)n)->value;
+ return ((art_leaf*)n)->values;
}
return NULL;
}
@@ -347,18 +347,19 @@ art_leaf* art_maximum(art_tree *t) {
return maximum((art_node*)t->root);
}
-static art_leaf* make_leaf(const unsigned char *key, int key_len, int score, void *value) {
+static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
art_leaf *l = (art_leaf *) malloc(sizeof(art_leaf) + key_len);
- l->value = value;
- l->score = (uint16_t) score;
+ l->values = new art_values;
+ l->values->ids.append_sorted(document->id);
+ l->score = std::max(l->score, document->score);
l->key_len = key_len;
memcpy(l->key, key, key_len);
return l;
}
-static int longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {
+static uint32_t longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {
int max_cmp = min(l1->key_len, l2->key_len) - depth;
- int idx;
+ uint32_t idx;
for (idx=0; idx < max_cmp; idx++) {
if (l1->key[depth+idx] != l2->key[depth+idx])
return idx;
@@ -523,10 +524,10 @@ static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_
return idx;
}
-static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, int key_len, int score, void *value, int depth, int *old) {
+static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, uint32_t key_len, art_document *document, int depth, int *old) {
// If we are at a NULL node, inject a leaf
if (!n) {
- *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, score, value));
+ *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, document));
return NULL;
}
@@ -537,9 +538,9 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Check if we are updating an existing value
if (!leaf_matches(l, key, key_len, depth)) {
*old = 1;
- void *old_val = l->value;
- l->value = value;
- l->score = (uint16_t) score;
+ art_values *old_val = l->values;
+ l->values->ids.append_sorted(document->id);
+ l->score = std::max(l->score, document->score);
return old_val;
}
@@ -547,12 +548,12 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
art_node4 *new_n = (art_node4*)alloc_node(NODE4);
// Create a new leaf
- art_leaf *l2 = make_leaf(key, key_len, score, value);
+ art_leaf *l2 = make_leaf(key, key_len, document);
// Determine longest prefix
- int longest_prefix = longest_common_prefix(l, l2, depth);
+ uint32_t longest_prefix = longest_common_prefix(l, l2, depth);
new_n->n.partial_len = longest_prefix;
- new_n->n.score = (uint16_t) std::max(l->score, (const uint16_t &) score);;
+ new_n->n.score = (uint16_t) std::max(l->score, (const uint16_t &) document->score);;
memcpy(new_n->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));
// Add the leafs to the new node4
*ref = (art_node*)new_n;
@@ -561,7 +562,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
return NULL;
}
- n->score = (uint16_t) std::max(n->score, (const uint16_t &) score);
+ n->score = (uint16_t) std::max(n->score, (const uint16_t &) document->score);
// Check if given node has a prefix
if (n->partial_len) {
@@ -576,7 +577,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
art_node4 *new_n = (art_node4*)alloc_node(NODE4);
*ref = (art_node*)new_n;
new_n->n.partial_len = prefix_diff;
- new_n->n.score = (uint16_t) std::max(n->score, (const uint16_t &) score);
+ new_n->n.score = (uint16_t) std::max(n->score, (const uint16_t &) document->score);
memcpy(new_n->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff));
// Adjust the prefix of the old node
@@ -594,7 +595,7 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
}
// Insert the new leaf
- art_leaf *l = make_leaf(key, key_len, score, value);
+ art_leaf *l = make_leaf(key, key_len, document);
add_child4(new_n, ref, key[depth+prefix_diff], SET_LEAF(l));
return NULL;
}
@@ -604,11 +605,11 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
// Find a child to recurse to
art_node **child = find_child(n, key[depth]);
if (child) {
- return recursive_insert(*child, child, key, key_len, score, value, depth+1, old);
+ return recursive_insert(*child, child, key, key_len, document, depth+1, old);
}
// No child, node goes within us
- art_leaf *l = make_leaf(key, key_len, score, value);
+ art_leaf *l = make_leaf(key, key_len, document);
add_child(n, ref, key[depth], SET_LEAF(l));
return NULL;
}
@@ -622,9 +623,10 @@ static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *
* @return NULL if the item was newly inserted, otherwise
* the old value pointer is returned.
*/
-void* art_insert(art_tree *t, const unsigned char *key, int key_len, int score, void *value) {
+void* art_insert(art_tree *t, const unsigned char *key, int key_len, art_document* document) {
int old_val = 0;
- void *old = recursive_insert(t->root, &t->root, key, key_len, score, value, 0, &old_val);
+
+ void *old = recursive_insert(t->root, &t->root, key, key_len, document, 0, &old_val);
if (!old_val) t->size++;
return old;
}
@@ -792,7 +794,7 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {
art_leaf *l = recursive_delete(t->root, &t->root, key, key_len, 0);
if (l) {
t->size--;
- void *old = l->value;
+ void *old = l->values;
free(l);
return old;
}
@@ -822,7 +824,7 @@ static int topk_iter(art_node *root) {
return lscore < rscore;
};
std::priority_queue<art_node *, std::vector<art_node *>, decltype(cmp)> q(cmp);
- std::vector<unsigned char*> results;
+ std::vector<art_leaf*> results;
q.push(root);
std::cout << "ROOT SCORE: " << root->score << std::endl;
@@ -834,7 +836,7 @@ static int topk_iter(art_node *root) {
if (!n) continue;
if (IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
- results.push_back(l->key);
+ results.push_back(l);
continue;
}
@@ -887,9 +889,9 @@ static int topk_iter(art_node *root) {
printf("OUTSIDE topk_iter: results size: %d\n", results.size());
- for(auto res: results) {
- std::cout << ">>>>!Key: " << res;
- std::cout << ", Value: " << std::endl;
+ for(auto leaf: results) {
+ std::cout << ">>>>!Key: " << leaf->key;
+ std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;
}
return 0;
@@ -902,7 +904,7 @@ static int recursive_iter(art_node *n, art_callback cb, void *data) {
if (IS_LEAF(n)) {
art_leaf *l = (art_leaf *) LEAF_RAW(n);
//printf("REC LEAF len: %d, key: %s\n", l->key_len, l->key);
- return cb(data, (const unsigned char*)l->key, l->key_len, l->value);
+ return cb(data, (const unsigned char*)l->key, l->key_len, l->values);
}
//printf("INTERNAL LEAF children: %d, partial_len: %d, partial: %s\n", n->num_children, n->partial_len, n->partial);
@@ -1015,7 +1017,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call
// Check if the expanded path matches
if (!leaf_prefix_matches((art_leaf*)n, key, key_len)) {
art_leaf *l = (art_leaf*)n;
- return cb(data, (const unsigned char*)l->key, l->key_len, l->value);
+ return cb(data, (const unsigned char*)l->key, l->key_len, l->values);
}
return 0;
}
@@ -1118,8 +1120,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
if (IS_LEAF(n)) {
printf("IS_LEAF\n");
- n = (art_node *) LEAF_RAW(n);
- art_leaf *l = (art_leaf *) n;
+ art_leaf *l = (art_leaf *) LEAF_RAW(n);
int row_min = 0;
int current_row[term_len+1];
@@ -1144,7 +1145,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
}
if(current_row[term_len] <= max_cost) {
- return cb(data, (const unsigned char*)l->key, l->key_len, l->value);
+ topk_iter(n);
}
return 0;
diff --git a/src/main.cpp b/src/main.cpp
index 53b260d2..7763b26d 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -5,17 +5,18 @@
#include <chrono>
#include <vector>
#include <time.h>
-#include <ewah.h>
+#include <art.h>
#include "art.h"
#include "topster.h"
+#include "forarray.h"
#include "util.h"
using namespace std;
static int test_prefix_cb(void *data, const unsigned char *k, uint32_t k_len, void *val) {
- cout << ">>>>Key: ";
+ cout << "#>>>>Key: ";
printf("%.*s", k_len, k);
- cout << ", Value: " << (uintptr_t) val << endl;
+ cout << ", ID: " << ((art_values*)val)->ids.at(0) << endl;
return 0;
}
@@ -55,36 +56,13 @@ void benchmark_heap_array() {
cout << "Time taken: " << timeMillis << endl;
}
-void ewahIntersect() {
- EWAHBoolArray<uint32_t> b1;
- EWAHBoolArray<uint32_t> b2;
- EWAHBoolArray<uint32_t> b3;
-
- for (uint32_t i=0; i < 1000000; ++i) {
- b1.set(2*i);
- b2.set(3*i);
- }
-
- auto begin1 = std::chrono::high_resolution_clock::now();
-
- b1.logicaland(b2, b3);
-
- vector<size_t> arr = b2.toArray();
-
- long long int timeMillis1 = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin1).count();
-
- int32_t counter = 0;
- for(int i=0; i<arr.size(); i++) {
- counter++;
- }
-
- cout << endl << "TIME: " << timeMillis1 << " and counter is: " << counter << endl;
-}
-
-void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint32_t score) {
+void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint16_t score) {
for(auto & token: tokens){
std::transform(token.begin(), token.end(), token.begin(), ::tolower);
- art_insert(&t, (const unsigned char *) token.c_str(), token.length(), score, (void*)doc_id);
+ art_document document;
+ document.id = doc_id;
+ document.score = score;
+ art_insert(&t, (const unsigned char *) token.c_str(), token.length(), &document);
}
}
@@ -106,11 +84,11 @@ int main() {
num++;
}
- const unsigned char *prefix = (const unsigned char *) "wherre";
+ const unsigned char *prefix = (const unsigned char *) "propellants";
size_t prefix_len = strlen((const char *) prefix);
auto begin = std::chrono::high_resolution_clock::now();
- art_iter_fuzzy_prefix(&t, prefix, prefix_len, 1, test_prefix_cb, NULL);
+ art_iter_fuzzy_prefix(&t, prefix, prefix_len, 2, test_prefix_cb, NULL);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
// art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);