Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Single leaf addition (cvt).

kishorenc0dfdfcf0d24e
diff --git a/include/cvt.h b/include/cvt.h
index ca0840cb..032de3c4 100644
--- a/include/cvt.h
+++ b/include/cvt.h
@@ -1,12 +1,14 @@
+#pragma once
+
/* Compact Variable Trie
================================================================================================================
- ates, at, as, bet, to, tk
+ ates, at, as, but, tok, too
[ * ]
⁄ | \
- a b t
+ a b t-o
/ \ \ /\
- s t etØ k o
+ s t utØ k o
/ / \ / \
Ø esØ Ø Ø Ø
@@ -14,18 +16,20 @@
BASIC DESIGN
============
- * All siblings in the same block.
+ * All nodes in the tree level in the same block.
* Pointer to ONLY FIRST child node of each sibling.
* Each sibling node's children represented by their character/byte
* For root there are no siblings, so pointer only to `a` child.
+ * Each node can be a single-char prefix, multi-char prefix or leaf.
- ROOT -> [0|0|PTRA][3][a][b][t]
- PTRA -> [0|PTRS][2|PTRE][4|PTRK[2][s][t][2][e][t][2][k][o]
- PTRS -> [0|PTRØ][1|PTRT][1][Ø][2][e][Ø]
+ ROOT -> [0|PTRA][3][a][b][t]
+ PTRA -> [0|PTRS] [2|PTRU] [4|PTRK] [2][s][t] [2][u][t] [2][k][o]
+ PTRS -> [0|PTRØ][1|PTRE][1][Ø][2][e][Ø]
PTRT -> [0|L_PTRE][3|PTRØ][3][e][s][Ø][1][Ø] (path compression)
PTRØ -> [0|LEAF]
- [TYPE+OFFSET][PTR_1]..[NUM_CHILDREN][A][B]..[X]
+ [OFFSET][PTR][TYPE]..[NUM_CHILDREN][A][B]..[X]
+ [ 16 ][45][ 3 ] (64 bits)
2 bytes for type+offset
6 bytes for address
@@ -41,6 +45,9 @@
Use array to represent children
Read `num_children` bytes and do sequential search
+ Multi-char node (COMPRESSED node) will be packed as:
+ [num_prefix][prefixes][num_children][children]
+
Removal of [be]
1. Realloc contents of PTR1 by removing "e" from the nodes list
@@ -51,58 +58,51 @@
#include <cstdint>
#include <cstddef>
+#include "logger.h"
+struct cvt_leaf_t {
+ size_t value;
+};
-
+enum CVT_NODE {
+ INTERNAL = 0,
+ LEAF = 1,
+ COMPRESSED = 2,
+};
class CVTrie {
private:
size_t size;
-
- // [TYPE+OFFSET][PTR_1]... (8 bytes each)
- // [NUM_PREFIX][A][B]..[X] (upto 33 bytes)
uint8_t* root;
const uintptr_t PTR_MASK = ~(1ULL << 48ULL);
public:
- uint8_t* get_ptr(const uint8_t* tagged_ptr) {
- // Unfortunately, right shifting of signed integer for sign extension is implementation-defined
- // but works on all major compilers
- return (uint8_t*)( (intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL );
- }
+ CVTrie(): root(nullptr) {
- uint8_t* tag_ptr(const uint8_t* ptr, const uint64_t data) {
- return (uint8_t*)(((uintptr_t)ptr & PTR_MASK) | (data << 48ULL));
}
- uint16_t get_data(const uint16_t* ptr) {
- return (uintptr_t)(ptr) >> 56ULL;;
+ inline void* get_ptr(const void* tagged_ptr) {
+ // Right shift of signed integer for sign extension is implementation-defined but works on major compilers
+ return (void*)( ((intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL) & ~3 );
}
- void add(const unsigned char* key, const size_t length, void* value) {
- // If the key exists, augment the node, otherwise insert a new node
-
- size_t num_siblings = 1;
- uint8_t* buf = root;
- size_t key_index = 0;
-
- if(root == nullptr) {
- // trie is empty
- root = new uint8_t[8+1+1];
+ inline void* tag_ptr(const void* ptr, const uint16_t offset, const CVT_NODE node_type) {
+ return (void*)(((uintptr_t)ptr & PTR_MASK) | (uint64_t(offset) << 48ULL) | uint64_t(node_type));
+ }
- }
+ inline uint8_t get_node_type(const void* tagged_ptr) {
+ return (uintptr_t)(tagged_ptr) & 3;
+ }
- while(true) {
- // for each sibling
- for(auto sindex = 0; sindex < num_siblings; sindex++) {
- unsigned char c = key[key_index];
+ inline uint16_t get_offset(const void* ptr) {
+ return (uintptr_t)(ptr) >> 48ULL;
+ }
+ void* find(const char* key, const uint8_t length);
- }
- }
- }
+ bool add(const char* key, const uint8_t length, void* value);
};
diff --git a/src/cvt.cpp b/src/cvt.cpp
new file mode 100644
index 00000000..df377cae
--- /dev/null
+++ b/src/cvt.cpp
@@ -0,0 +1,73 @@
+#include <cvt.h>
+#include <cstring>
+#include "logger.h"
+
+bool CVTrie::add(const char *key, const uint8_t length, void *value) {
+ // If the key exists, augment the node, otherwise insert a new node
+
+ if(root == nullptr) {
+ // Trie is empty, so add a single leaf node:
+ // [0|PTRLEAF][3][f][o][o]
+
+ uint8_t* block = new uint8_t[8 + 1 + length];
+
+ void* node = tag_ptr(value, 0, LEAF);
+
+ std::memcpy(block, &node, sizeof(void*));
+ std::memcpy(block+sizeof(void*), &length, 1);
+ std::memcpy(block+sizeof(void*)+1, key, length);
+
+ root = block;
+ return true;
+ }
+
+ uint8_t node_type = get_node_type(root);
+
+ if(node_type == LEAF) {
+ // Compare new key with child key to identify common prefix
+ // e.g. welcome vs welding (or) we vs welcome (or) welcome vs foobar
+ }
+
+ /*size_t num_siblings = 1;
+ size_t key_index = 0;
+
+ while(true) {
+ // for each sibling
+ for(auto sindex = 0; sindex < num_siblings; sindex++) {
+ unsigned char c = key[key_index];
+
+
+ }
+ }*/
+}
+
+void *CVTrie::find(const char *key, const uint8_t length) {
+ size_t key_index = 0;
+ void* curr = nullptr;
+ std::memcpy(&curr, root, sizeof curr);
+
+ while(true) {
+ uint8_t node_type = get_node_type(curr);
+ if(node_type == LEAF) {
+ size_t key_len = *(uint8_t*)(root+sizeof(void*));
+
+ if(key_index+key_len != length) {
+ return nullptr;
+ }
+
+ for(size_t i=0; i<key_len; i++) {
+ uint8_t this_char = *(uint8_t *) (root + sizeof(void *) + 1 + i);
+ if(key[key_index++] != this_char) {
+ return nullptr;
+ }
+ }
+
+ void* leaf = get_ptr(curr);
+ return leaf;
+ }
+
+ break;
+ }
+
+ return nullptr;
+}
\ No newline at end of file
diff --git a/test/cvt_test.cpp b/test/cvt_test.cpp
new file mode 100644
index 00000000..8a2cbb2a
--- /dev/null
+++ b/test/cvt_test.cpp
@@ -0,0 +1,59 @@
+#include <gtest/gtest.h>
+#include <cvt.h>
+
+TEST(CVTTest, TaggedPointers) {
+ CVTrie trie;
+ uint8_t* bytes = new uint8_t[3];
+ bytes[0] = 0;
+ bytes[1] = 1;
+ bytes[2] = 2;
+
+ // LEAF
+
+ void* leaf_tagged_ptr = trie.tag_ptr(bytes, 100, LEAF);
+ ASSERT_EQ(LEAF, trie.get_node_type(leaf_tagged_ptr));
+ ASSERT_EQ(100, trie.get_offset(leaf_tagged_ptr));
+
+ uint8_t* leaf_data = static_cast<uint8_t *>(trie.get_ptr(leaf_tagged_ptr));
+
+ ASSERT_EQ(0, leaf_data[0]);
+ ASSERT_EQ(1, leaf_data[1]);
+ ASSERT_EQ(2, leaf_data[2]);
+
+ // INTERNAL
+
+ void* internal_tagged_ptr = trie.tag_ptr(bytes, 100, INTERNAL);
+ ASSERT_EQ(INTERNAL, trie.get_node_type(internal_tagged_ptr));
+ ASSERT_EQ(100, trie.get_offset(internal_tagged_ptr));
+
+ uint8_t* internal_data = static_cast<uint8_t *>(trie.get_ptr(internal_tagged_ptr));
+
+ ASSERT_EQ(0, internal_data[0]);
+ ASSERT_EQ(1, internal_data[1]);
+ ASSERT_EQ(2, internal_data[2]);
+
+ // COMPRESSED
+
+ void* compressed_tagged_ptr = trie.tag_ptr(bytes, 100, COMPRESSED);
+ ASSERT_EQ(COMPRESSED, trie.get_node_type(compressed_tagged_ptr));
+ ASSERT_EQ(100, trie.get_offset(compressed_tagged_ptr));
+
+ uint8_t* compressed_data = static_cast<uint8_t *>(trie.get_ptr(compressed_tagged_ptr));
+
+ ASSERT_EQ(0, compressed_data[0]);
+ ASSERT_EQ(1, compressed_data[1]);
+ ASSERT_EQ(2, compressed_data[2]);
+
+ delete [] bytes;
+}
+
+TEST(CVTTest, AddSingleLeaf) {
+ CVTrie trie;
+
+ cvt_leaf_t leaf{108};
+ ASSERT_TRUE(trie.add("foo", 3, &leaf));
+
+ ASSERT_EQ(&leaf, trie.find("foo", 3));
+ ASSERT_EQ(nullptr, trie.find("foooo", 5));
+ ASSERT_EQ(nullptr, trie.find("f", 1));
+}
\ No newline at end of file