Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Single leaf addition (cvt).
kishorenc0dfdfcf0d24e
diff --git a/include/cvt.h b/include/cvt.hindex ca0840cb..032de3c4 100644--- a/include/cvt.h+++ b/include/cvt.h@@ -1,12 +1,14 @@+#pragma once+/* Compact Variable Trie================================================================================================================- ates, at, as, bet, to, tk+ ates, at, as, but, tok, too[ * ]⁄ | \- a b t+ a b t-o/ \ \ /\- s t etØ k o+ s t utØ k o/ / \ / \Ø esØ Ø Ø Ø@@ -14,18 +16,20 @@BASIC DESIGN============- * All siblings in the same block.+ * All nodes in the tree level in the same block.* Pointer to ONLY FIRST child node of each sibling.* Each sibling node's children represented by their character/byte* For root there are no siblings, so pointer only to `a` child.+ * Each node can be a single-char prefix, multi-char prefix or leaf.- ROOT -> [0|0|PTRA][3][a][b][t]- PTRA -> [0|PTRS][2|PTRE][4|PTRK[2][s][t][2][e][t][2][k][o]- PTRS -> [0|PTRØ][1|PTRT][1][Ø][2][e][Ø]+ ROOT -> [0|PTRA][3][a][b][t]+ PTRA -> [0|PTRS] [2|PTRU] [4|PTRK] [2][s][t] [2][u][t] [2][k][o]+ PTRS -> [0|PTRØ][1|PTRE][1][Ø][2][e][Ø]PTRT -> [0|L_PTRE][3|PTRØ][3][e][s][Ø][1][Ø] (path compression)PTRØ -> [0|LEAF]- [TYPE+OFFSET][PTR_1]..[NUM_CHILDREN][A][B]..[X]+ [OFFSET][PTR][TYPE]..[NUM_CHILDREN][A][B]..[X]+ [ 16 ][45][ 3 ] (64 bits)2 bytes for type+offset6 bytes for address@@ -41,6 +45,9 @@Use array to represent childrenRead `num_children` bytes and do sequential search+ Multi-char node (COMPRESSED node) will be packed as:+ [num_prefix][prefixes][num_children][children]+Removal of [be]1. Realloc contents of PTR1 by removing "e" from the nodes list@@ -51,58 +58,51 @@#include <cstdint>#include <cstddef>+#include "logger.h"+struct cvt_leaf_t {+ size_t value;+};-+enum CVT_NODE {+ INTERNAL = 0,+ LEAF = 1,+ COMPRESSED = 2,+};class CVTrie {private:size_t size;-- // [TYPE+OFFSET][PTR_1]... (8 bytes each)- // [NUM_PREFIX][A][B]..[X] (upto 33 bytes)uint8_t* root;const uintptr_t PTR_MASK = ~(1ULL << 48ULL);public:- uint8_t* get_ptr(const uint8_t* tagged_ptr) {- // Unfortunately, right shifting of signed integer for sign extension is implementation-defined- // but works on all major compilers- return (uint8_t*)( (intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL );- }+ CVTrie(): root(nullptr) {- uint8_t* tag_ptr(const uint8_t* ptr, const uint64_t data) {- return (uint8_t*)(((uintptr_t)ptr & PTR_MASK) | (data << 48ULL));}- uint16_t get_data(const uint16_t* ptr) {- return (uintptr_t)(ptr) >> 56ULL;;+ inline void* get_ptr(const void* tagged_ptr) {+ // Right shift of signed integer for sign extension is implementation-defined but works on major compilers+ return (void*)( ((intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL) & ~3 );}- void add(const unsigned char* key, const size_t length, void* value) {- // If the key exists, augment the node, otherwise insert a new node-- size_t num_siblings = 1;- uint8_t* buf = root;- size_t key_index = 0;-- if(root == nullptr) {- // trie is empty- root = new uint8_t[8+1+1];+ inline void* tag_ptr(const void* ptr, const uint16_t offset, const CVT_NODE node_type) {+ return (void*)(((uintptr_t)ptr & PTR_MASK) | (uint64_t(offset) << 48ULL) | uint64_t(node_type));+ }- }+ inline uint8_t get_node_type(const void* tagged_ptr) {+ return (uintptr_t)(tagged_ptr) & 3;+ }- while(true) {- // for each sibling- for(auto sindex = 0; sindex < num_siblings; sindex++) {- unsigned char c = key[key_index];+ inline uint16_t get_offset(const void* ptr) {+ return (uintptr_t)(ptr) >> 48ULL;+ }+ void* find(const char* key, const uint8_t length);- }- }- }+ bool add(const char* key, const uint8_t length, void* value);};diff --git a/src/cvt.cpp b/src/cvt.cppnew file mode 100644index 00000000..df377cae--- /dev/null+++ b/src/cvt.cpp@@ -0,0 +1,73 @@+#include <cvt.h>+#include <cstring>+#include "logger.h"++bool CVTrie::add(const char *key, const uint8_t length, void *value) {+ // If the key exists, augment the node, otherwise insert a new node++ if(root == nullptr) {+ // Trie is empty, so add a single leaf node:+ // [0|PTRLEAF][3][f][o][o]++ uint8_t* block = new uint8_t[8 + 1 + length];++ void* node = tag_ptr(value, 0, LEAF);++ std::memcpy(block, &node, sizeof(void*));+ std::memcpy(block+sizeof(void*), &length, 1);+ std::memcpy(block+sizeof(void*)+1, key, length);++ root = block;+ return true;+ }++ uint8_t node_type = get_node_type(root);++ if(node_type == LEAF) {+ // Compare new key with child key to identify common prefix+ // e.g. welcome vs welding (or) we vs welcome (or) welcome vs foobar+ }++ /*size_t num_siblings = 1;+ size_t key_index = 0;++ while(true) {+ // for each sibling+ for(auto sindex = 0; sindex < num_siblings; sindex++) {+ unsigned char c = key[key_index];+++ }+ }*/+}++void *CVTrie::find(const char *key, const uint8_t length) {+ size_t key_index = 0;+ void* curr = nullptr;+ std::memcpy(&curr, root, sizeof curr);++ while(true) {+ uint8_t node_type = get_node_type(curr);+ if(node_type == LEAF) {+ size_t key_len = *(uint8_t*)(root+sizeof(void*));++ if(key_index+key_len != length) {+ return nullptr;+ }++ for(size_t i=0; i<key_len; i++) {+ uint8_t this_char = *(uint8_t *) (root + sizeof(void *) + 1 + i);+ if(key[key_index++] != this_char) {+ return nullptr;+ }+ }++ void* leaf = get_ptr(curr);+ return leaf;+ }++ break;+ }++ return nullptr;+}\ No newline at end of filediff --git a/test/cvt_test.cpp b/test/cvt_test.cppnew file mode 100644index 00000000..8a2cbb2a--- /dev/null+++ b/test/cvt_test.cpp@@ -0,0 +1,59 @@+#include <gtest/gtest.h>+#include <cvt.h>++TEST(CVTTest, TaggedPointers) {+ CVTrie trie;+ uint8_t* bytes = new uint8_t[3];+ bytes[0] = 0;+ bytes[1] = 1;+ bytes[2] = 2;++ // LEAF++ void* leaf_tagged_ptr = trie.tag_ptr(bytes, 100, LEAF);+ ASSERT_EQ(LEAF, trie.get_node_type(leaf_tagged_ptr));+ ASSERT_EQ(100, trie.get_offset(leaf_tagged_ptr));++ uint8_t* leaf_data = static_cast<uint8_t *>(trie.get_ptr(leaf_tagged_ptr));++ ASSERT_EQ(0, leaf_data[0]);+ ASSERT_EQ(1, leaf_data[1]);+ ASSERT_EQ(2, leaf_data[2]);++ // INTERNAL++ void* internal_tagged_ptr = trie.tag_ptr(bytes, 100, INTERNAL);+ ASSERT_EQ(INTERNAL, trie.get_node_type(internal_tagged_ptr));+ ASSERT_EQ(100, trie.get_offset(internal_tagged_ptr));++ uint8_t* internal_data = static_cast<uint8_t *>(trie.get_ptr(internal_tagged_ptr));++ ASSERT_EQ(0, internal_data[0]);+ ASSERT_EQ(1, internal_data[1]);+ ASSERT_EQ(2, internal_data[2]);++ // COMPRESSED++ void* compressed_tagged_ptr = trie.tag_ptr(bytes, 100, COMPRESSED);+ ASSERT_EQ(COMPRESSED, trie.get_node_type(compressed_tagged_ptr));+ ASSERT_EQ(100, trie.get_offset(compressed_tagged_ptr));++ uint8_t* compressed_data = static_cast<uint8_t *>(trie.get_ptr(compressed_tagged_ptr));++ ASSERT_EQ(0, compressed_data[0]);+ ASSERT_EQ(1, compressed_data[1]);+ ASSERT_EQ(2, compressed_data[2]);++ delete [] bytes;+}++TEST(CVTTest, AddSingleLeaf) {+ CVTrie trie;++ cvt_leaf_t leaf{108};+ ASSERT_TRUE(trie.add("foo", 3, &leaf));++ ASSERT_EQ(&leaf, trie.find("foo", 3));+ ASSERT_EQ(nullptr, trie.find("foooo", 5));+ ASSERT_EQ(nullptr, trie.find("f", 1));+}\ No newline at end of file