Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Initial commit. Fuzzy prefix match works.

Kishore Nallan64f53b6420b8
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..f1d51bec
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.idea
+*.iml
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 00000000..bcdc5f8d
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 2.8)
+project(search)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+
+include_directories(include)
+#file(GLOB SOURCES "src/*.cpp")
+
+add_executable(search src/art.c src/main.cpp)
\ No newline at end of file
diff --git a/include/art.h b/include/art.h
new file mode 100644
index 00000000..64e8ae90
--- /dev/null
+++ b/include/art.h
@@ -0,0 +1,224 @@
+#include <stdint.h>
+#include <stdbool.h>
+
+#ifndef ART_H
+#define ART_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define NODE4 1
+#define NODE16 2
+#define NODE48 3
+#define NODE256 4
+
+#define MAX_PREFIX_LEN 10
+
+#if defined(__GNUC__) && !defined(__clang__)
+# if __STDC_VERSION__ >= 199901L && 402 == (__GNUC__ * 100 + __GNUC_MINOR__)
+/*
+ * GCC 4.2.2's C99 inline keyword support is pretty broken; avoid. Introduced in
+ * GCC 4.2.something, fixed in 4.3.0. So checking for specific major.minor of
+ * 4.2 is fine.
+ */
+# define BROKEN_GCC_C99_INLINE
+# endif
+#endif
+
+typedef int(*art_callback)(void *data, const unsigned char *key, uint32_t key_len, void *value);
+
+/**
+ * This struct is included as part
+ * of all the various node sizes
+ */
+typedef struct {
+ uint8_t type;
+ uint8_t num_children;
+ uint32_t partial_len;
+ unsigned char partial[MAX_PREFIX_LEN];
+} art_node;
+
+/**
+ * Small node with only 4 children
+ */
+typedef struct {
+ art_node n;
+ unsigned char keys[4];
+ art_node *children[4];
+} art_node4;
+
+/**
+ * Node with 16 children
+ */
+typedef struct {
+ art_node n;
+ unsigned char keys[16];
+ art_node *children[16];
+} art_node16;
+
+/**
+ * Node with 48 children, but
+ * a full 256 byte field.
+ */
+typedef struct {
+ art_node n;
+ unsigned char keys[256];
+ art_node *children[48];
+} art_node48;
+
+/**
+ * Full node with 256 children
+ */
+typedef struct {
+ art_node n;
+ art_node *children[256];
+} art_node256;
+
+/**
+ * Represents a leaf. These are
+ * of arbitrary size, as they include the key.
+ */
+typedef struct {
+ void *value;
+ uint32_t key_len;
+ unsigned char key[];
+} art_leaf;
+
+/**
+ * Main struct, points to root.
+ */
+typedef struct {
+ art_node *root;
+ uint64_t size;
+} art_tree;
+
+/**
+ * Initializes an ART tree
+ * @return 0 on success.
+ */
+int art_tree_init(art_tree *t);
+
+/**
+ * DEPRECATED
+ * Initializes an ART tree
+ * @return 0 on success.
+ */
+#define init_art_tree(...) art_tree_init(__VA_ARGS__)
+
+/**
+ * Destroys an ART tree
+ * @return 0 on success.
+ */
+int art_tree_destroy(art_tree *t);
+
+/**
+ * DEPRECATED
+ * Initializes an ART tree
+ * @return 0 on success.
+ */
+#define destroy_art_tree(...) art_tree_destroy(__VA_ARGS__)
+
+/**
+ * Returns the size of the ART tree.
+ */
+#ifdef BROKEN_GCC_C99_INLINE
+# define art_size(t) ((t)->size)
+#else
+inline uint64_t art_size(art_tree *t) {
+ return t->size;
+}
+#endif
+
+/**
+ * Inserts a new value into the ART tree
+ * @arg t The tree
+ * @arg key The key
+ * @arg key_len The length of the key
+ * @arg value Opaque value.
+ * @return NULL if the item was newly inserted, otherwise
+ * the old value pointer is returned.
+ */
+void* art_insert(art_tree *t, const unsigned char *key, int key_len, void *value);
+
+/**
+ * Deletes a value from the ART tree
+ * @arg t The tree
+ * @arg key The key
+ * @arg key_len The length of the key
+ * @return NULL if the item was not found, otherwise
+ * the value pointer is returned.
+ */
+void* art_delete(art_tree *t, const unsigned char *key, int key_len);
+
+/**
+ * Searches for a value in the ART tree
+ * @arg t The tree
+ * @arg key The key
+ * @arg key_len The length of the key
+ * @return NULL if the item was not found, otherwise
+ * the value pointer is returned.
+ */
+void* art_search(const art_tree *t, const unsigned char *key, int key_len);
+
+/**
+ * Returns the minimum valued leaf
+ * @return The minimum leaf or NULL
+ */
+art_leaf* art_minimum(art_tree *t);
+
+/**
+ * Returns the maximum valued leaf
+ * @return The maximum leaf or NULL
+ */
+art_leaf* art_maximum(art_tree *t);
+
+/**
+ * Iterates through the entries pairs in the map,
+ * invoking a callback for each. The call back gets a
+ * key, value for each and returns an integer stop value.
+ * If the callback returns non-zero, then the iteration stops.
+ * @arg t The tree to iterate over
+ * @arg cb The callback function to invoke
+ * @arg data Opaque handle passed to the callback
+ * @return 0 on success, or the return of the callback.
+ */
+int art_iter(art_tree *t, art_callback cb, void *data);
+
+/**
+ * Iterates through the entries pairs in the map,
+ * invoking a callback for each that matches a given prefix.
+ * The call back gets a key, value for each and returns an integer stop value.
+ * If the callback returns non-zero, then the iteration stops.
+ * @arg t The tree to iterate over
+ * @arg prefix The prefix of keys to read
+ * @arg prefix_len The length of the prefix
+ * @arg cb The callback function to invoke
+ * @arg data Opaque handle passed to the callback
+ * @return 0 on success, or the return of the callback.
+ */
+int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, art_callback cb, void *data);
+
+/**
+ * Iterates through the entries pairs in the map,
+ * invoking a callback for each that matches a given prefix within a fuzzy distance of 2.
+ * The call back gets a key, value for each and returns an integer stop value.
+ * If the callback returns non-zero, then the iteration stops.
+ * @arg t The tree to iterate over
+ * @arg prefix The prefix of keys to read
+ * @arg prefix_len The length of the prefix
+ * @arg cb The callback function to invoke
+ * @arg data Opaque handle passed to the callback
+ * @return 0 on success, or the return of the callback.
+ */
+int art_iter_fuzzy_prefix(art_tree *t, const unsigned char *term, int term_len, int max_cost, art_callback cb,
+ void *data);
+
+static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term, int term_len, int tidx,
+ int pidx, int max_cost, int cost_so_far, art_callback cb, void *data);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/art.c b/src/art.c
new file mode 100644
index 00000000..526c0195
--- /dev/null
+++ b/src/art.c
@@ -0,0 +1,1075 @@
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <stdio.h>
+#include <emmintrin.h>
+#include <assert.h>
+#include <art.h>
+#include "art.h"
+
+/**
+ * Macros to manipulate pointer tags
+ */
+#define IS_LEAF(x) (((uintptr_t)x & 1))
+#define SET_LEAF(x) ((void*)((uintptr_t)x | 1))
+#define LEAF_RAW(x) ((void*)((uintptr_t)x & ~1))
+
+#define MIN3(a, b, c) ((a) < (b) ? ((a) < (c) ? (a) : (c)) : ((b) < (c) ? (b) : (c)))
+
+#define IGNORE_PRINTF 1
+
+#ifdef IGNORE_PRINTF
+#define printf(fmt, ...) (0)
+#endif
+
+static int levenshtein(const unsigned char *s1, unsigned int s1len, const unsigned char *s2, unsigned int s2len) {
+ unsigned int x, y, lastdiag, olddiag;
+ unsigned int column[s1len+1];
+ for (y = 1; y <= s1len; y++)
+ column[y] = y;
+ for (x = 1; x <= s2len; x++) {
+ column[0] = x;
+ for (y = 1, lastdiag = x-1; y <= s1len; y++) {
+ olddiag = column[y];
+ column[y] = MIN3(column[y] + 1, column[y-1] + 1, lastdiag + (s1[y-1] == s2[x-1] ? 0 : 1));
+ lastdiag = olddiag;
+ }
+ }
+ return(column[s1len]);
+}
+
+/**
+ * Allocates a node of the given type,
+ * initializes to zero and sets the type.
+ */
+static art_node* alloc_node(uint8_t type) {
+ art_node* n;
+ switch (type) {
+ case NODE4:
+ n = calloc(1, sizeof(art_node4));
+ break;
+ case NODE16:
+ n = calloc(1, sizeof(art_node16));
+ break;
+ case NODE48:
+ n = calloc(1, sizeof(art_node48));
+ break;
+ case NODE256:
+ n = calloc(1, sizeof(art_node256));
+ break;
+ default:
+ abort();
+ }
+ n->type = type;
+ return n;
+}
+
+/**
+ * Initializes an ART tree
+ * @return 0 on success.
+ */
+int art_tree_init(art_tree *t) {
+ t->root = NULL;
+ t->size = 0;
+ return 0;
+}
+
+// Recursively destroys the tree
+static void destroy_node(art_node *n) {
+ // Break if null
+ if (!n) return;
+
+ // Special case leafs
+ if (IS_LEAF(n)) {
+ free(LEAF_RAW(n));
+ return;
+ }
+
+ // Handle each node type
+ int i;
+ union {
+ art_node4 *p1;
+ art_node16 *p2;
+ art_node48 *p3;
+ art_node256 *p4;
+ } p;
+ switch (n->type) {
+ case NODE4:
+ p.p1 = (art_node4*)n;
+ for (i=0;i<n->num_children;i++) {
+ destroy_node(p.p1->children[i]);
+ }
+ break;
+
+ case NODE16:
+ p.p2 = (art_node16*)n;
+ for (i=0;i<n->num_children;i++) {
+ destroy_node(p.p2->children[i]);
+ }
+ break;
+
+ case NODE48:
+ p.p3 = (art_node48*)n;
+ for (i=0;i<n->num_children;i++) {
+ destroy_node(p.p3->children[i]);
+ }
+ break;
+
+ case NODE256:
+ p.p4 = (art_node256*)n;
+ for (i=0;i<256;i++) {
+ if (p.p4->children[i])
+ destroy_node(p.p4->children[i]);
+ }
+ break;
+
+ default:
+ abort();
+ }
+
+ // Free ourself on the way up
+ free(n);
+}
+
+/**
+ * Destroys an ART tree
+ * @return 0 on success.
+ */
+int art_tree_destroy(art_tree *t) {
+ destroy_node(t->root);
+ return 0;
+}
+
+/**
+ * Returns the size of the ART tree.
+ */
+
+#ifndef BROKEN_GCC_C99_INLINE
+extern inline uint64_t art_size(art_tree *t);
+#endif
+
+static art_node** find_child(art_node *n, unsigned char c) {
+ int i, mask, bitfield;
+ union {
+ art_node4 *p1;
+ art_node16 *p2;
+ art_node48 *p3;
+ art_node256 *p4;
+ } p;
+ switch (n->type) {
+ case NODE4:
+ p.p1 = (art_node4*)n;
+ for (i=0;i < n->num_children; i++) {
+ if (p.p1->keys[i] == c)
+ return &p.p1->children[i];
+ }
+ break;
+
+ {
+ __m128i cmp;
+ case NODE16:
+ p.p2 = (art_node16*)n;
+
+ // Compare the key to all 16 stored keys
+ cmp = _mm_cmpeq_epi8(_mm_set1_epi8(c),
+ _mm_loadu_si128((__m128i*)p.p2->keys));
+
+ // Use a mask to ignore children that don't exist
+ mask = (1 << n->num_children) - 1;
+ bitfield = _mm_movemask_epi8(cmp) & mask;
+
+ /*
+ * If we have a match (any bit set) then we can
+ * return the pointer match using ctz to get
+ * the index.
+ */
+ if (bitfield)
+ return &p.p2->children[__builtin_ctz(bitfield)];
+ break;
+ }
+
+ case NODE48:
+ p.p3 = (art_node48*)n;
+ i = p.p3->keys[c];
+ if (i)
+ return &p.p3->children[i-1];
+ break;
+
+ case NODE256:
+ p.p4 = (art_node256*)n;
+ if (p.p4->children[c])
+ return &p.p4->children[c];
+ break;
+
+ default:
+ abort();
+ }
+ return NULL;
+}
+
+// Simple inlined if
+static inline int min(int a, int b) {
+ return (a < b) ? a : b;
+}
+
+/**
+ * Returns the number of prefix characters shared between
+ * the key and node.
+ */
+static int check_prefix(const art_node *n, const unsigned char *key, int key_len, int depth) {
+ int max_cmp = min(min(n->partial_len, MAX_PREFIX_LEN), key_len - depth);
+ int idx;
+ for (idx=0; idx < max_cmp; idx++) {
+ if (n->partial[idx] != key[depth+idx])
+ return idx;
+ }
+ return idx;
+}
+
+/**
+ * Checks if a leaf matches
+ * @return 0 on success.
+ */
+static int leaf_matches(const art_leaf *n, const unsigned char *key, int key_len, int depth) {
+ (void)depth;
+ // Fail if the key lengths are different
+ if (n->key_len != (uint32_t)key_len) return 1;
+
+ // Compare the keys starting at the depth
+ return memcmp(n->key, key, key_len);
+}
+
+/**
+ * Searches for a value in the ART tree
+ * @arg t The tree
+ * @arg key The key
+ * @arg key_len The length of the key
+ * @return NULL if the item was not found, otherwise
+ * the value pointer is returned.
+ */
+void* art_search(const art_tree *t, const unsigned char *key, int key_len) {
+ art_node **child;
+ art_node *n = t->root;
+ int prefix_len, depth = 0;
+ while (n) {
+ // Might be a leaf
+ if (IS_LEAF(n)) {
+ n = LEAF_RAW(n);
+ // Check if the expanded path matches
+ if (!leaf_matches((art_leaf*)n, key, key_len, depth)) {
+ return ((art_leaf*)n)->value;
+ }
+ return NULL;
+ }
+
+ // Bail if the prefix does not match
+ if (n->partial_len) {
+ prefix_len = check_prefix(n, key, key_len, depth);
+ if (prefix_len != min(MAX_PREFIX_LEN, n->partial_len))
+ return NULL;
+ depth = depth + n->partial_len;
+ }
+
+ // Recursively search
+ child = find_child(n, key[depth]);
+ n = (child) ? *child : NULL;
+ depth++;
+ }
+ return NULL;
+}
+
+// Find the minimum leaf under a node
+static art_leaf* minimum(const art_node *n) {
+ // Handle base cases
+ if (!n) return NULL;
+ if (IS_LEAF(n)) return LEAF_RAW(n);
+
+ int idx;
+ switch (n->type) {
+ case NODE4:
+ return minimum(((art_node4*)n)->children[0]);
+ case NODE16:
+ return minimum(((art_node16*)n)->children[0]);
+ case NODE48:
+ idx=0;
+ while (!((art_node48*)n)->keys[idx]) idx++;
+ idx = ((art_node48*)n)->keys[idx] - 1;
+ return minimum(((art_node48*)n)->children[idx]);
+ case NODE256:
+ idx=0;
+ while (!((art_node256*)n)->children[idx]) idx++;
+ return minimum(((art_node256*)n)->children[idx]);
+ default:
+ abort();
+ }
+}
+
+// Find the maximum leaf under a node
+static art_leaf* maximum(const art_node *n) {
+ // Handle base cases
+ if (!n) return NULL;
+ if (IS_LEAF(n)) return LEAF_RAW(n);
+
+ int idx;
+ switch (n->type) {
+ case NODE4:
+ return maximum(((art_node4*)n)->children[n->num_children-1]);
+ case NODE16:
+ return maximum(((art_node16*)n)->children[n->num_children-1]);
+ case NODE48:
+ idx=255;
+ while (!((art_node48*)n)->keys[idx]) idx--;
+ idx = ((art_node48*)n)->keys[idx] - 1;
+ return maximum(((art_node48*)n)->children[idx]);
+ case NODE256:
+ idx=255;
+ while (!((art_node256*)n)->children[idx]) idx--;
+ return maximum(((art_node256*)n)->children[idx]);
+ default:
+ abort();
+ }
+}
+
+/**
+ * Returns the minimum valued leaf
+ */
+art_leaf* art_minimum(art_tree *t) {
+ return minimum((art_node*)t->root);
+}
+
+/**
+ * Returns the maximum valued leaf
+ */
+art_leaf* art_maximum(art_tree *t) {
+ return maximum((art_node*)t->root);
+}
+
+static art_leaf* make_leaf(const unsigned char *key, int key_len, void *value) {
+ art_leaf *l = malloc(sizeof(art_leaf)+key_len);
+ l->value = value;
+ l->key_len = key_len;
+ memcpy(l->key, key, key_len);
+ return l;
+}
+
+static int longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth) {
+ int max_cmp = min(l1->key_len, l2->key_len) - depth;
+ int idx;
+ for (idx=0; idx < max_cmp; idx++) {
+ if (l1->key[depth+idx] != l2->key[depth+idx])
+ return idx;
+ }
+ return idx;
+}
+
+static void copy_header(art_node *dest, art_node *src) {
+ dest->num_children = src->num_children;
+ dest->partial_len = src->partial_len;
+ memcpy(dest->partial, src->partial, min(MAX_PREFIX_LEN, src->partial_len));
+}
+
+static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child) {
+ (void)ref;
+ n->n.num_children++;
+ n->children[c] = child;
+}
+
+static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *child) {
+ if (n->n.num_children < 48) {
+ int pos = 0;
+ while (n->children[pos]) pos++;
+ n->children[pos] = child;
+ n->keys[c] = pos + 1;
+ n->n.num_children++;
+ } else {
+ art_node256 *new = (art_node256*)alloc_node(NODE256);
+ for (int i=0;i<256;i++) {
+ if (n->keys[i]) {
+ new->children[i] = n->children[n->keys[i] - 1];
+ }
+ }
+ copy_header((art_node*)new, (art_node*)n);
+ *ref = (art_node*)new;
+ free(n);
+ add_child256(new, ref, c, child);
+ }
+}
+
+static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *child) {
+ if (n->n.num_children < 16) {
+ __m128i cmp;
+
+ // Compare the key to all 16 stored keys
+ cmp = _mm_cmplt_epi8(_mm_set1_epi8(c),
+ _mm_loadu_si128((__m128i*)n->keys));
+
+ // Use a mask to ignore children that don't exist
+ unsigned mask = (1 << n->n.num_children) - 1;
+ unsigned bitfield = _mm_movemask_epi8(cmp) & mask;
+
+ // Check if less than any
+ unsigned idx;
+ if (bitfield) {
+ idx = __builtin_ctz(bitfield);
+ memmove(n->keys+idx+1,n->keys+idx,n->n.num_children-idx);
+ memmove(n->children+idx+1,n->children+idx,
+ (n->n.num_children-idx)*sizeof(void*));
+ } else
+ idx = n->n.num_children;
+
+ // Set the child
+ n->keys[idx] = c;
+ n->children[idx] = child;
+ n->n.num_children++;
+
+ } else {
+ art_node48 *new = (art_node48*)alloc_node(NODE48);
+
+ // Copy the child pointers and populate the key map
+ memcpy(new->children, n->children,
+ sizeof(void*)*n->n.num_children);
+ for (int i=0;i<n->n.num_children;i++) {
+ new->keys[n->keys[i]] = i + 1;
+ }
+ copy_header((art_node*)new, (art_node*)n);
+ *ref = (art_node*)new;
+ free(n);
+ add_child48(new, ref, c, child);
+ }
+}
+
+static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *child) {
+ if (n->n.num_children < 4) {
+ int idx;
+ for (idx=0; idx < n->n.num_children; idx++) {
+ if (c < n->keys[idx]) break;
+ }
+
+ // Shift to make room
+ memmove(n->keys+idx+1, n->keys+idx, n->n.num_children - idx);
+ memmove(n->children+idx+1, n->children+idx,
+ (n->n.num_children - idx)*sizeof(void*));
+
+ // Insert element
+ n->keys[idx] = c;
+ n->children[idx] = child;
+ n->n.num_children++;
+
+ } else {
+ art_node16 *new = (art_node16*)alloc_node(NODE16);
+
+ // Copy the child pointers and the key map
+ memcpy(new->children, n->children,
+ sizeof(void*)*n->n.num_children);
+ memcpy(new->keys, n->keys,
+ sizeof(unsigned char)*n->n.num_children);
+ copy_header((art_node*)new, (art_node*)n);
+ *ref = (art_node*)new;
+ free(n);
+ add_child16(new, ref, c, child);
+ }
+}
+
+static void add_child(art_node *n, art_node **ref, unsigned char c, void *child) {
+ switch (n->type) {
+ case NODE4:
+ return add_child4((art_node4*)n, ref, c, child);
+ case NODE16:
+ return add_child16((art_node16*)n, ref, c, child);
+ case NODE48:
+ return add_child48((art_node48*)n, ref, c, child);
+ case NODE256:
+ return add_child256((art_node256*)n, ref, c, child);
+ default:
+ abort();
+ }
+}
+
+/**
+ * Calculates the index at which the prefixes mismatch
+ */
+static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_len, int depth) {
+ int max_cmp = min(min(MAX_PREFIX_LEN, n->partial_len), key_len - depth);
+ int idx;
+ for (idx=0; idx < max_cmp; idx++) {
+ if (n->partial[idx] != key[depth+idx])
+ return idx;
+ }
+
+ // If the prefix is short we can avoid finding a leaf
+ if (n->partial_len > MAX_PREFIX_LEN) {
+ // Prefix is longer than what we've checked, find a leaf
+ art_leaf *l = minimum(n);
+ max_cmp = min(l->key_len, key_len)- depth;
+ for (; idx < max_cmp; idx++) {
+ if (l->key[idx+depth] != key[depth+idx])
+ return idx;
+ }
+ }
+ return idx;
+}
+
+static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, int key_len, void *value, int depth, int *old) {
+ // If we are at a NULL node, inject a leaf
+ if (!n) {
+ *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, value));
+ return NULL;
+ }
+
+ // If we are at a leaf, we need to replace it with a node
+ if (IS_LEAF(n)) {
+ art_leaf *l = LEAF_RAW(n);
+
+ // Check if we are updating an existing value
+ if (!leaf_matches(l, key, key_len, depth)) {
+ *old = 1;
+ void *old_val = l->value;
+ l->value = value;
+ return old_val;
+ }
+
+ // New value, we must split the leaf into a node4
+ art_node4 *new = (art_node4*)alloc_node(NODE4);
+
+ // Create a new leaf
+ art_leaf *l2 = make_leaf(key, key_len, value);
+
+ // Determine longest prefix
+ int longest_prefix = longest_common_prefix(l, l2, depth);
+ new->n.partial_len = longest_prefix;
+ memcpy(new->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix));
+ // Add the leafs to the new node4
+ *ref = (art_node*)new;
+ add_child4(new, ref, l->key[depth+longest_prefix], SET_LEAF(l));
+ add_child4(new, ref, l2->key[depth+longest_prefix], SET_LEAF(l2));
+ return NULL;
+ }
+
+ // Check if given node has a prefix
+ if (n->partial_len) {
+ // Determine if the prefixes differ, since we need to split
+ int prefix_diff = prefix_mismatch(n, key, key_len, depth);
+ if ((uint32_t)prefix_diff >= n->partial_len) {
+ depth += n->partial_len;
+ goto RECURSE_SEARCH;
+ }
+
+ // Create a new node
+ art_node4 *new = (art_node4*)alloc_node(NODE4);
+ *ref = (art_node*)new;
+ new->n.partial_len = prefix_diff;
+ memcpy(new->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff));
+
+ // Adjust the prefix of the old node
+ if (n->partial_len <= MAX_PREFIX_LEN) {
+ add_child4(new, ref, n->partial[prefix_diff], n);
+ n->partial_len -= (prefix_diff+1);
+ memmove(n->partial, n->partial+prefix_diff+1,
+ min(MAX_PREFIX_LEN, n->partial_len));
+ } else {
+ n->partial_len -= (prefix_diff+1);
+ art_leaf *l = minimum(n);
+ add_child4(new, ref, l->key[depth+prefix_diff], n);
+ memcpy(n->partial, l->key+depth+prefix_diff+1,
+ min(MAX_PREFIX_LEN, n->partial_len));
+ }
+
+ // Insert the new leaf
+ art_leaf *l = make_leaf(key, key_len, value);
+ add_child4(new, ref, key[depth+prefix_diff], SET_LEAF(l));
+ return NULL;
+ }
+
+ RECURSE_SEARCH:;
+
+ // Find a child to recurse to
+ art_node **child = find_child(n, key[depth]);
+ if (child) {
+ return recursive_insert(*child, child, key, key_len, value, depth+1, old);
+ }
+
+ // No child, node goes within us
+ art_leaf *l = make_leaf(key, key_len, value);
+ add_child(n, ref, key[depth], SET_LEAF(l));
+ return NULL;
+}
+
+/**
+ * Inserts a new value into the ART tree
+ * @arg t The tree
+ * @arg key The key
+ * @arg key_len The length of the key
+ * @arg value Opaque value.
+ * @return NULL if the item was newly inserted, otherwise
+ * the old value pointer is returned.
+ */
+void* art_insert(art_tree *t, const unsigned char *key, int key_len, void *value) {
+ int old_val = 0;
+ void *old = recursive_insert(t->root, &t->root, key, key_len, value, 0, &old_val);
+ if (!old_val) t->size++;
+ return old;
+}
+
+static void remove_child256(art_node256 *n, art_node **ref, unsigned char c) {
+ n->children[c] = NULL;
+ n->n.num_children--;
+
+ // Resize to a node48 on underflow, not immediately to prevent
+ // trashing if we sit on the 48/49 boundary
+ if (n->n.num_children == 37) {
+ art_node48 *new = (art_node48*)alloc_node(NODE48);
+ *ref = (art_node*)new;
+ copy_header((art_node*)new, (art_node*)n);
+
+ int pos = 0;
+ for (int i=0;i<256;i++) {
+ if (n->children[i]) {
+ new->children[pos] = n->children[i];
+ new->keys[i] = pos + 1;
+ pos++;
+ }
+ }
+ free(n);
+ }
+}
+
+static void remove_child48(art_node48 *n, art_node **ref, unsigned char c) {
+ int pos = n->keys[c];
+ n->keys[c] = 0;
+ n->children[pos-1] = NULL;
+ n->n.num_children--;
+
+ if (n->n.num_children == 12) {
+ art_node16 *new = (art_node16*)alloc_node(NODE16);
+ *ref = (art_node*)new;
+ copy_header((art_node*)new, (art_node*)n);
+
+ int child = 0;
+ for (int i=0;i<256;i++) {
+ pos = n->keys[i];
+ if (pos) {
+ new->keys[child] = i;
+ new->children[child] = n->children[pos - 1];
+ child++;
+ }
+ }
+ free(n);
+ }
+}
+
+static void remove_child16(art_node16 *n, art_node **ref, art_node **l) {
+ int pos = l - n->children;
+ memmove(n->keys+pos, n->keys+pos+1, n->n.num_children - 1 - pos);
+ memmove(n->children+pos, n->children+pos+1, (n->n.num_children - 1 - pos)*sizeof(void*));
+ n->n.num_children--;
+
+ if (n->n.num_children == 3) {
+ art_node4 *new = (art_node4*)alloc_node(NODE4);
+ *ref = (art_node*)new;
+ copy_header((art_node*)new, (art_node*)n);
+ memcpy(new->keys, n->keys, 4);
+ memcpy(new->children, n->children, 4*sizeof(void*));
+ free(n);
+ }
+}
+
+static void remove_child4(art_node4 *n, art_node **ref, art_node **l) {
+ int pos = l - n->children;
+ memmove(n->keys+pos, n->keys+pos+1, n->n.num_children - 1 - pos);
+ memmove(n->children+pos, n->children+pos+1, (n->n.num_children - 1 - pos)*sizeof(void*));
+ n->n.num_children--;
+
+ // Remove nodes with only a single child
+ if (n->n.num_children == 1) {
+ art_node *child = n->children[0];
+ if (!IS_LEAF(child)) {
+ // Concatenate the prefixes
+ int prefix = n->n.partial_len;
+ if (prefix < MAX_PREFIX_LEN) {
+ n->n.partial[prefix] = n->keys[0];
+ prefix++;
+ }
+ if (prefix < MAX_PREFIX_LEN) {
+ int sub_prefix = min(child->partial_len, MAX_PREFIX_LEN - prefix);
+ memcpy(n->n.partial+prefix, child->partial, sub_prefix);
+ prefix += sub_prefix;
+ }
+
+ // Store the prefix in the child
+ memcpy(child->partial, n->n.partial, min(prefix, MAX_PREFIX_LEN));
+ child->partial_len += n->n.partial_len + 1;
+ }
+ *ref = child;
+ free(n);
+ }
+}
+
+static void remove_child(art_node *n, art_node **ref, unsigned char c, art_node **l) {
+ switch (n->type) {
+ case NODE4:
+ return remove_child4((art_node4*)n, ref, l);
+ case NODE16:
+ return remove_child16((art_node16*)n, ref, l);
+ case NODE48:
+ return remove_child48((art_node48*)n, ref, c);
+ case NODE256:
+ return remove_child256((art_node256*)n, ref, c);
+ default:
+ abort();
+ }
+}
+
+static art_leaf* recursive_delete(art_node *n, art_node **ref, const unsigned char *key, int key_len, int depth) {
+ // Search terminated
+ if (!n) return NULL;
+
+ // Handle hitting a leaf node
+ if (IS_LEAF(n)) {
+ art_leaf *l = LEAF_RAW(n);
+ if (!leaf_matches(l, key, key_len, depth)) {
+ *ref = NULL;
+ return l;
+ }
+ return NULL;
+ }
+
+ // Bail if the prefix does not match
+ if (n->partial_len) {
+ int prefix_len = check_prefix(n, key, key_len, depth);
+ if (prefix_len != min(MAX_PREFIX_LEN, n->partial_len)) {
+ return NULL;
+ }
+ depth = depth + n->partial_len;
+ }
+
+ // Find child node
+ art_node **child = find_child(n, key[depth]);
+ if (!child) return NULL;
+
+ // If the child is leaf, delete from this node
+ if (IS_LEAF(*child)) {
+ art_leaf *l = LEAF_RAW(*child);
+ if (!leaf_matches(l, key, key_len, depth)) {
+ remove_child(n, ref, key[depth], child);
+ return l;
+ }
+ return NULL;
+
+ // Recurse
+ } else {
+ return recursive_delete(*child, child, key, key_len, depth+1);
+ }
+}
+
+/**
+ * Deletes a value from the ART tree
+ * @arg t The tree
+ * @arg key The key
+ * @arg key_len The length of the key
+ * @return NULL if the item was not found, otherwise
+ * the value pointer is returned.
+ */
+void* art_delete(art_tree *t, const unsigned char *key, int key_len) {
+ art_leaf *l = recursive_delete(t->root, &t->root, key, key_len, 0);
+ if (l) {
+ t->size--;
+ void *old = l->value;
+ free(l);
+ return old;
+ }
+ return NULL;
+}
+
+// Recursively iterates over the tree
+static int recursive_iter(art_node *n, art_callback cb, void *data) {
+ // Handle base cases
+ if (!n) return 0;
+ if (IS_LEAF(n)) {
+ art_leaf *l = LEAF_RAW(n);
+ //printf("REC LEAF len: %d, key: %s\n", l->key_len, l->key);
+ return cb(data, (const unsigned char*)l->key, l->key_len, l->value);
+ }
+
+ //printf("INTERNAL LEAF children: %d, partial_len: %d, partial: %s\n", n->num_children, n->partial_len, n->partial);
+
+ int idx, res;
+ switch (n->type) {
+ case NODE4:
+ for (int i=0; i < n->num_children; i++) {
+ //printf("INTERNAL LEAF key[i]: %c\n", ((art_node4*)n)->keys[i]);
+ res = recursive_iter(((art_node4*)n)->children[i], cb, data);
+ if (res) return res;
+ }
+ break;
+
+ case NODE16:
+ for (int i=0; i < n->num_children; i++) {
+ res = recursive_iter(((art_node16*)n)->children[i], cb, data);
+ if (res) return res;
+ }
+ break;
+
+ case NODE48:
+ for (int i=0; i < 256; i++) {
+ idx = ((art_node48*)n)->keys[i];
+ if (!idx) continue;
+
+ res = recursive_iter(((art_node48*)n)->children[idx-1], cb, data);
+ if (res) return res;
+ }
+ break;
+
+ case NODE256:
+ for (int i=0; i < 256; i++) {
+ if (!((art_node256*)n)->children[i]) continue;
+ res = recursive_iter(((art_node256*)n)->children[i], cb, data);
+ if (res) return res;
+ }
+ break;
+
+ default:
+ abort();
+ }
+ return 0;
+}
+
+/**
+ * Iterates through the entries pairs in the map,
+ * invoking a callback for each. The call back gets a
+ * key, value for each and returns an integer stop value.
+ * If the callback returns non-zero, then the iteration stops.
+ * @arg t The tree to iterate over
+ * @arg cb The callback function to invoke
+ * @arg data Opaque handle passed to the callback
+ * @return 0 on success, or the return of the callback.
+ */
+int art_iter(art_tree *t, art_callback cb, void *data) {
+ return recursive_iter(t->root, cb, data);
+}
+
+/**
+ * Checks if a leaf prefix matches
+ * @return 0 on success.
+ */
+static int leaf_prefix_matches(const art_leaf *n, const unsigned char *prefix, int prefix_len) {
+ // Fail if the key length is too short
+ if (n->key_len < (uint32_t)prefix_len) return 1;
+
+ // Compare the keys
+ return memcmp(n->key, prefix, prefix_len);
+}
+
+/**
+ * Check if a leaf prefix matches the key within a given threshold
+ * @return true if the fuzzy match succeeds, false otherwise
+ */
+static int leaf_prefix_mismatch(const art_leaf *l, const unsigned char *term, int term_len, int tidx) {
+ // We have to compare the full key here since we are at the leaf
+ int min_len = min(term_len, l->key_len);
+ return min_len == 0 ? 0 : levenshtein(l->key, min_len, term, min_len);
+}
+
+
+/**
+ * Iterates through the entries pairs in the map,
+ * invoking a callback for each that matches a given prefix.
+ * The call back gets a key, value for each and returns an integer stop value.
+ * If the callback returns non-zero, then the iteration stops.
+ * @arg t The tree to iterate over
+ * @arg prefix The prefix of keys to read
+ * @arg prefix_len The length of the prefix
+ * @arg cb The callback function to invoke
+ * @arg data Opaque handle passed to the callback
+ * @return 0 on success, or the return of the callback.
+ */
+int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_callback cb, void *data) {
+ art_node **child;
+ art_node *n = t->root;
+ int prefix_len, depth = 0;
+ while (n) {
+ //printf("partial_len: %d\n", n->num_children);
+
+ // Might be a leaf
+ if (IS_LEAF(n)) {
+ n = LEAF_RAW(n);
+
+ printf("RAW LEAF len: %d, children: %d\n", n->partial_len, n->num_children);
+
+ // Check if the expanded path matches
+ if (!leaf_prefix_matches((art_leaf*)n, key, key_len)) {
+ art_leaf *l = (art_leaf*)n;
+ return cb(data, (const unsigned char*)l->key, l->key_len, l->value);
+ }
+ return 0;
+ }
+
+ printf("IS INTERNAL\n");
+ printf("Prefix len: %d, children: %d, value: %s\n", n->partial_len, n->num_children, n->partial);
+
+ // If the depth matches the prefix, we need to handle this node
+ if (depth == key_len) {
+ art_leaf *l = minimum(n);
+ printf("DEPTH LEAF len: %d, key: %s\n", l->key_len, l->key);
+
+ if (!leaf_prefix_matches(l, key, key_len))
+ return recursive_iter(n, cb, data);
+ return 0;
+ }
+
+ // Bail if the prefix does not match
+ if (n->partial_len) {
+ prefix_len = prefix_mismatch(n, key, key_len, depth);
+
+ // If there is no match, search is terminated
+ if (!prefix_len)
+ return 0;
+
+ // If we've matched the prefix, iterate on this node
+ else if (depth + prefix_len == key_len) {
+ return recursive_iter(n, cb, data);
+ }
+
+ // if there is a full match, go deeper
+ depth = depth + n->partial_len;
+ }
+
+ // Recursively search
+ child = find_child(n, key[depth]);
+ n = (child) ? *child : NULL;
+ depth++;
+ }
+ return 0;
+}
+
+#define comp_child_char_recurse(node, cmp_char) {\
+ if(term[tidx] == cmp_char) {\
+ art_iter_fuzzy_prefix_recurse(node, term, term_len, tidx+1, 0, max_cost, cost_so_far, cb, data);\
+ } else {\
+ cost_so_far++;\
+ art_iter_fuzzy_prefix_recurse(node, term, term_len, tidx, 0, max_cost, cost_so_far, cb, data);\
+ art_iter_fuzzy_prefix_recurse(node, term, term_len, tidx+1, 0, max_cost, cost_so_far, cb, data);\
+ art_iter_fuzzy_prefix_recurse(node, term, term_len, tidx+2, 0, max_cost, cost_so_far, cb, data);\
+ }\
+}\
+
+static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term, int term_len, int tidx,
+ int pidx, int max_cost, int cost_so_far, art_callback cb, void *data) {
+ /*
+ Pass a "cost_so_far" through the recursive function along with an index of current position within the word.
+ If the letter at that position doesn't match the current trie node's letter then adjust the "cost so far"
+ by one and search the next level with the index the same (a delete), index+1 (a replace), and index+2 (an insert).
+ Otherwise don't adjust the cost-so-far, and just continue on searching index+1.
+
+ Also add a condition to see if the node word thus far is larger than the original search term by more than
+ the max cost, and short-circuit the search at that point if it is
+ */
+
+ //printf("partial_len: %d\n", n->num_children);
+
+ if(cost_so_far > max_cost) return 0;
+
+ if(tidx >= term_len) {
+ // return this node - it could be either child or internal
+ printf("CALLLING recursive_iter: cost_so_far: %d\n", cost_so_far);
+ return recursive_iter(n, cb, NULL);
+ }
+
+ if (IS_LEAF(n)) {
+ printf("IS LEAF\n");
+ n = LEAF_RAW(n);
+
+ // check if the key matches term within given cost_so_far
+ art_leaf *l = (art_leaf *) n;
+ int mismatch_len = leaf_prefix_mismatch(l, term, term_len, tidx);
+ cost_so_far += mismatch_len;
+
+ printf("LEAF tidx: %d, mismatch_len: %d, term: %s, cost_so_far: %d, key_len: %d, key: %s\n", tidx, mismatch_len, term, cost_so_far, l->key_len, l->key);
+ printf("max_cost: %d\n", max_cost);
+
+ if (cost_so_far <= max_cost) {
+ art_leaf *l = (art_leaf*)n;
+ printf("FROM MAIN CB\n");
+ return cb(data, (const unsigned char*)l->key, l->key_len, l->value);
+ }
+
+ return 0;
+ }
+
+ // Internal node - recalculate cost based on partial
+
+ if(pidx == min(MAX_PREFIX_LEN, n->partial_len)) {
+ // end of partial - look to children
+
+ printf("END OF INTERNAL\n");
+ printf("tidx: %d, partial_len: %d, children: %d, value: %s\n", tidx, n->partial_len, n->num_children, n->partial);
+
+ switch (n->type) {
+ case NODE4:
+ for (int i=0; i < n->num_children; i++) {
+ char child_char = ((art_node4*)n)->keys[i];
+ art_node* child = ((art_node4*)n)->children[i];
+ printf("child_char: %c\n", child_char);
+ comp_child_char_recurse(child, child_char);
+ }
+ break;
+
+ case NODE16:
+ for (int i=0; i < n->num_children; i++) {
+ char child_char = ((art_node16*)n)->keys[i];
+ art_node* child = ((art_node16*)n)->children[i];
+ comp_child_char_recurse(child, child_char);
+ }
+ break;
+
+ case NODE48:
+ for (int i=0; i < 256; i++) {
+ char ix = ((art_node48*)n)->keys[i];
+ if (!ix) continue;
+
+ char child_char = ((art_node48*)n)->keys[ix - 1];
+ art_node* child = ((art_node48*)n)->children[ix - 1];
+ comp_child_char_recurse(child, child_char);
+ }
+ break;
+
+ case NODE256:
+ for (int i=0; i < 256; i++) {
+ if (!((art_node256*)n)->children[i]) continue;
+ art_node* child = ((art_node256*)n)->children[i];
+ art_iter_fuzzy_prefix_recurse(child, term, term_len, tidx, 0, max_cost, cost_so_far, cb, data);
+ }
+ break;
+
+ default:
+ abort();
+ }
+
+ return 0;
+ }
+
+ printf("IS INTERNAL\n");
+ printf("cost_so_far: %d, tidx: %d, term[tidx]: %c, partial[pidx]: %c, pidx: %d, partial: %s partial_len: %d, term_len: %d\n", cost_so_far, tidx, term[tidx], n->partial[pidx], pidx, n->partial, n->partial_len, term_len);
+
+ // handle partial
+ comp_child_char_recurse(n, n->partial[pidx]);
+ return 0;
+}
+
+/**
+ * Iterates through the entries pairs in the map,
+ * invoking a callback for each that matches a given prefix within a fuzzy distance of 2.
+ * The call back gets a key, value for each and returns an integer stop value.
+ * If the callback returns non-zero, then the iteration stops.
+ * @arg t The tree to iterate over
+ * @arg prefix The prefix of keys to read
+ * @arg prefix_len The length of the prefix
+ * @arg max_cost Maximum fuzzy edit distance
+ * @arg cb The callback function to invoke
+ * @arg data Opaque handle passed to the callback
+ * @return 0 on success, or the return of the callback.
+ */
+int art_iter_fuzzy_prefix(art_tree *t, const unsigned char *term, int term_len,
+ int max_cost, art_callback cb, void *data) {
+
+ return art_iter_fuzzy_prefix_recurse(t->root, term, term_len, 0, 0, max_cost, 0, cb, data);
+}
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
new file mode 100644
index 00000000..b56a0180
--- /dev/null
+++ b/src/main.cpp
@@ -0,0 +1,39 @@
+#include <iostream>
+#include <check.h>
+#include "art.h"
+#include <fstream>
+
+using namespace std;
+
+static int test_prefix_cb(void *data, const unsigned char *k, uint32_t k_len, void *val) {
+ cout << ">>>>Key: " << k << ", Value: " << (uintptr_t) val << endl;
+ return 0;
+}
+
+
+int main() {
+ cout << "Running demo...\n\n";
+
+ art_tree t;
+ art_tree_init(&t);
+
+ std::ifstream infile("/Users/kishorenc/others/wreally/search/test/words.txt");
+
+ std::string line;
+ uintptr_t num = 1;
+
+ while (std::getline(infile, line)) {
+ //cout << "Line: " << line << ", number = " << line << endl;
+ art_insert(&t, (const unsigned char *) line.c_str(), line.length(), (void*)num);
+ num++;
+ }
+
+ const unsigned char *prefix = (const unsigned char *) "hel";
+ art_iter_fuzzy_prefix(&t, prefix, strlen((const char *) prefix), 2, test_prefix_cb, NULL);
+// art_iter_prefix(&t, prefix, strlen((const char *) prefix), test_prefix_cb, NULL);
+
+// art_iter(&t, test_prefix_cb, NULL);
+
+ art_tree_destroy(&t);
+ return 0;
+}
\ No newline at end of file
diff --git a/test/words.txt b/test/words.txt
new file mode 100644
index 00000000..ea01ca24
--- /dev/null
+++ b/test/words.txt
@@ -0,0 +1,4 @@
+hello
+hat
+hell
+shawshank
\ No newline at end of file