Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor `NumericTrie::Node`, use `ids_t` instead of `sorted_array`. Refactor `indexable_limit` function.
Harpreet Sangarab4e60d5b868
diff --git a/include/index.h b/include/index.hindex 70771b3e..9370ac31 100644--- a/include/index.h+++ b/include/index.h@@ -30,7 +30,7 @@#include "vector_query_ops.h"#include "hnswlib/hnswlib.h"#include "filter.h"-#include "numeric_range_trie_test.h"+#include "numeric_range_trie.h"static constexpr size_t ARRAY_FACET_DIM = 4;using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;diff --git a/include/numeric_range_trie_test.h b/include/numeric_range_trie.hsimilarity index 97%rename from include/numeric_range_trie_test.hrename to include/numeric_range_trie.hindex 8b7bd22c..4e5dc628 100644--- a/include/numeric_range_trie_test.h+++ b/include/numeric_range_trie.h@@ -1,7 +1,6 @@#pragma once-#include <map>-#include "sorted_array.h"+#include <ids_t.h>constexpr short EXPANSE = 256;@@ -10,7 +9,7 @@ class NumericTrie {class Node {Node** children = nullptr;- sorted_array seq_ids;+ void* seq_ids = SET_COMPACT_IDS(compact_id_list_t::create(0, {}));void insert_helper(const int64_t& value, const uint32_t& seq_id, char& level, const char& max_level);@@ -30,6 +29,8 @@ class NumericTrie {public:~Node() {+ ids_t::destroy_list(seq_ids);+if (children != nullptr) {for (auto i = 0; i < EXPANSE; i++) {delete children[i];diff --git a/src/numeric_range_trie.cpp b/src/numeric_range_trie.cppindex 66676e93..400eb94b 100644--- a/src/numeric_range_trie.cpp+++ b/src/numeric_range_trie.cpp@@ -1,7 +1,6 @@#include <timsort.hpp>#include <set>-#include <complex>-#include "numeric_range_trie_test.h"+#include "numeric_range_trie.h"#include "array_utils.h"void NumericTrie::insert(const int64_t& value, const uint32_t& seq_id) {@@ -407,12 +406,31 @@ NumericTrie::iterator_t NumericTrie::search_equal_to(const int64_t& value) {return NumericTrie::iterator_t(matches);}-inline uint64_t indexable_limit(const char& max_level) {- return std::pow(EXPANSE, max_level);+inline int64_t indexable_limit(const char& max_level) {+ switch (max_level) {+ case 1:+ return 0xFF;+ case 2:+ return 0xFFFF;+ case 3:+ return 0xFFFFFF;+ case 4:+ return 0xFFFFFFFF;+ case 5:+ return 0xFFFFFFFFFF;+ case 6:+ return 0xFFFFFFFFFFFF;+ case 7:+ return 0xFFFFFFFFFFFFFF;+ case 8:+ return 0x7FFFFFFFFFFFFFFF;+ default:+ return 0;+ }}void NumericTrie::Node::insert(const int64_t& value, const uint32_t& seq_id, const char& max_level) {- if (value >= indexable_limit(max_level)) {+ if (value > indexable_limit(max_level)) {return;}@@ -442,7 +460,7 @@ inline int get_geopoint_index(const uint64_t& cell_id, const char& level) {}void NumericTrie::Node::remove(const int64_t& value, const uint32_t& id, const char& max_level) {- if (value >= indexable_limit(max_level)) {+ if (value > indexable_limit(max_level)) {return;}@@ -451,7 +469,7 @@ void NumericTrie::Node::remove(const int64_t& value, const uint32_t& id, const cauto index = get_index(value, level, max_level);while (level < max_level) {- root->seq_ids.remove_value(id);+ ids_t::erase(root->seq_ids, id);if (root->children == nullptr || root->children[index] == nullptr) {return;@@ -461,12 +479,12 @@ void NumericTrie::Node::remove(const int64_t& value, const uint32_t& id, const cindex = get_index(value, ++level, max_level);}- root->seq_ids.remove_value(id);+ ids_t::erase(root->seq_ids, id);if (root->children != nullptr && root->children[index] != nullptr) {auto& child = root->children[index];- child->seq_ids.remove_value(id);- if (child->seq_ids.getLength() == 0) {+ ids_t::erase(child->seq_ids, id);+ if (ids_t::num_ids(child->seq_ids) == 0) {delete child;child = nullptr;}@@ -479,9 +497,7 @@ void NumericTrie::Node::insert_helper(const int64_t& value, const uint32_t& seq_}// Root node contains all the sequence ids present in the tree.- if (!seq_ids.contains(seq_id)) {- seq_ids.append(seq_id);- }+ ids_t::upsert(seq_ids, seq_id);if (++level <= max_level) {if (children == nullptr) {@@ -504,9 +520,7 @@ void NumericTrie::Node::insert_geopoint_helper(const uint64_t& cell_id, const ui}// Root node contains all the sequence ids present in the tree.- if (!seq_ids.contains(seq_id)) {- seq_ids.append(seq_id);- }+ ids_t::upsert(seq_ids, seq_id);if (++level <= max_level) {if (children == nullptr) {@@ -562,12 +576,7 @@ void NumericTrie::Node::search_geopoints(const std::vector<uint64_t>& cell_ids,}for (auto const& match: matches) {- auto const& m_seq_ids = match->seq_ids.uncompress();- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {- geo_result_ids.push_back(m_seq_ids[i]);- }-- delete [] m_seq_ids;+ ids_t::uncompress(match->seq_ids, geo_result_ids);}gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());@@ -580,7 +589,7 @@ void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, coauto index = get_geopoint_index(cell_id, level);while (level < max_level) {- root->seq_ids.remove_value(id);+ ids_t::erase(root->seq_ids, id);if (root->children == nullptr || root->children[index] == nullptr) {return;@@ -590,12 +599,12 @@ void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, coindex = get_geopoint_index(cell_id, ++level);}- root->seq_ids.remove_value(id);+ ids_t::erase(root->seq_ids, id);if (root->children != nullptr && root->children[index] != nullptr) {auto& child = root->children[index];- child->seq_ids.remove_value(id);- if (child->seq_ids.getLength() == 0) {+ ids_t::erase(child->seq_ids, id);+ if (ids_t::num_ids(child->seq_ids) == 0) {delete child;child = nullptr;}@@ -603,8 +612,8 @@ void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, co}void NumericTrie::Node::get_all_ids(uint32_t*& ids, uint32_t& ids_length) {- ids = seq_ids.uncompress();- ids_length = seq_ids.getLength();+ ids = ids_t::uncompress(seq_ids);+ ids_length = ids_t::num_ids(seq_ids);}void NumericTrie::Node::search_less_than(const int64_t& value, const char& max_level,@@ -620,12 +629,7 @@ void NumericTrie::Node::search_less_than(const int64_t& value, const char& max_lstd::vector<uint32_t> consolidated_ids;for (auto const& match: matches) {- auto const& m_seq_ids = match->seq_ids.uncompress();- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {- consolidated_ids.push_back(m_seq_ids[i]);- }-- delete [] m_seq_ids;+ ids_t::uncompress(match->seq_ids, consolidated_ids);}gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());@@ -673,17 +677,12 @@ void NumericTrie::Node::search_range(const int64_t& low, const int64_t& high, coreturn;}std::vector<NumericTrie::Node*> matches;- search_range_helper(low, high >= indexable_limit(max_level) ? (int64_t) indexable_limit(max_level) - 1 : high,+ search_range_helper(low, high >= indexable_limit(max_level) ? indexable_limit(max_level) : high,max_level, matches);std::vector<uint32_t> consolidated_ids;for (auto const& match: matches) {- auto const& m_seq_ids = match->seq_ids.uncompress();- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {- consolidated_ids.push_back(m_seq_ids[i]);- }-- delete [] m_seq_ids;+ ids_t::uncompress(match->seq_ids, consolidated_ids);}gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());@@ -768,12 +767,7 @@ void NumericTrie::Node::search_greater_than(const int64_t& value, const char& mastd::vector<uint32_t> consolidated_ids;for (auto const& match: matches) {- auto const& m_seq_ids = match->seq_ids.uncompress();- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {- consolidated_ids.push_back(m_seq_ids[i]);- }-- delete [] m_seq_ids;+ ids_t::uncompress(match->seq_ids, consolidated_ids);}gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());@@ -817,7 +811,7 @@ void NumericTrie::Node::search_greater_than_helper(const int64_t& value, char& lvoid NumericTrie::Node::search_equal_to(const int64_t& value, const char& max_level,uint32_t*& ids, uint32_t& ids_length) {- if (value >= indexable_limit(max_level)) {+ if (value > indexable_limit(max_level)) {return;}diff --git a/test/numeric_range_trie_test.cpp b/test/numeric_range_trie_test.cppindex a53347ad..92bf01cc 100644--- a/test/numeric_range_trie_test.cpp+++ b/test/numeric_range_trie_test.cpp@@ -1,7 +1,7 @@#include <gtest/gtest.h>#include <collection_manager.h>#include "collection.h"-#include "numeric_range_trie_test.h"+#include "numeric_range_trie.h"class NumericRangeTrieTest : public ::testing::Test {protected: