Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor `NumericTrie::Node`, use `ids_t` instead of `sorted_array`. Refactor `indexable_limit` function.

Harpreet Sangarab4e60d5b868
diff --git a/include/index.h b/include/index.h
index 70771b3e..9370ac31 100644
--- a/include/index.h
+++ b/include/index.h
@@ -30,7 +30,7 @@
#include "vector_query_ops.h"
#include "hnswlib/hnswlib.h"
#include "filter.h"
-#include "numeric_range_trie_test.h"
+#include "numeric_range_trie.h"
static constexpr size_t ARRAY_FACET_DIM = 4;
using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;
diff --git a/include/numeric_range_trie_test.h b/include/numeric_range_trie.h
similarity index 97%
rename from include/numeric_range_trie_test.h
rename to include/numeric_range_trie.h
index 8b7bd22c..4e5dc628 100644
--- a/include/numeric_range_trie_test.h
+++ b/include/numeric_range_trie.h
@@ -1,7 +1,6 @@
#pragma once
-#include <map>
-#include "sorted_array.h"
+#include <ids_t.h>
constexpr short EXPANSE = 256;
@@ -10,7 +9,7 @@ class NumericTrie {
class Node {
Node** children = nullptr;
- sorted_array seq_ids;
+ void* seq_ids = SET_COMPACT_IDS(compact_id_list_t::create(0, {}));
void insert_helper(const int64_t& value, const uint32_t& seq_id, char& level, const char& max_level);
@@ -30,6 +29,8 @@ class NumericTrie {
public:
~Node() {
+ ids_t::destroy_list(seq_ids);
+
if (children != nullptr) {
for (auto i = 0; i < EXPANSE; i++) {
delete children[i];
diff --git a/src/numeric_range_trie.cpp b/src/numeric_range_trie.cpp
index 66676e93..400eb94b 100644
--- a/src/numeric_range_trie.cpp
+++ b/src/numeric_range_trie.cpp
@@ -1,7 +1,6 @@
#include <timsort.hpp>
#include <set>
-#include <complex>
-#include "numeric_range_trie_test.h"
+#include "numeric_range_trie.h"
#include "array_utils.h"
void NumericTrie::insert(const int64_t& value, const uint32_t& seq_id) {
@@ -407,12 +406,31 @@ NumericTrie::iterator_t NumericTrie::search_equal_to(const int64_t& value) {
return NumericTrie::iterator_t(matches);
}
-inline uint64_t indexable_limit(const char& max_level) {
- return std::pow(EXPANSE, max_level);
+inline int64_t indexable_limit(const char& max_level) {
+ switch (max_level) {
+ case 1:
+ return 0xFF;
+ case 2:
+ return 0xFFFF;
+ case 3:
+ return 0xFFFFFF;
+ case 4:
+ return 0xFFFFFFFF;
+ case 5:
+ return 0xFFFFFFFFFF;
+ case 6:
+ return 0xFFFFFFFFFFFF;
+ case 7:
+ return 0xFFFFFFFFFFFFFF;
+ case 8:
+ return 0x7FFFFFFFFFFFFFFF;
+ default:
+ return 0;
+ }
}
void NumericTrie::Node::insert(const int64_t& value, const uint32_t& seq_id, const char& max_level) {
- if (value >= indexable_limit(max_level)) {
+ if (value > indexable_limit(max_level)) {
return;
}
@@ -442,7 +460,7 @@ inline int get_geopoint_index(const uint64_t& cell_id, const char& level) {
}
void NumericTrie::Node::remove(const int64_t& value, const uint32_t& id, const char& max_level) {
- if (value >= indexable_limit(max_level)) {
+ if (value > indexable_limit(max_level)) {
return;
}
@@ -451,7 +469,7 @@ void NumericTrie::Node::remove(const int64_t& value, const uint32_t& id, const c
auto index = get_index(value, level, max_level);
while (level < max_level) {
- root->seq_ids.remove_value(id);
+ ids_t::erase(root->seq_ids, id);
if (root->children == nullptr || root->children[index] == nullptr) {
return;
@@ -461,12 +479,12 @@ void NumericTrie::Node::remove(const int64_t& value, const uint32_t& id, const c
index = get_index(value, ++level, max_level);
}
- root->seq_ids.remove_value(id);
+ ids_t::erase(root->seq_ids, id);
if (root->children != nullptr && root->children[index] != nullptr) {
auto& child = root->children[index];
- child->seq_ids.remove_value(id);
- if (child->seq_ids.getLength() == 0) {
+ ids_t::erase(child->seq_ids, id);
+ if (ids_t::num_ids(child->seq_ids) == 0) {
delete child;
child = nullptr;
}
@@ -479,9 +497,7 @@ void NumericTrie::Node::insert_helper(const int64_t& value, const uint32_t& seq_
}
// Root node contains all the sequence ids present in the tree.
- if (!seq_ids.contains(seq_id)) {
- seq_ids.append(seq_id);
- }
+ ids_t::upsert(seq_ids, seq_id);
if (++level <= max_level) {
if (children == nullptr) {
@@ -504,9 +520,7 @@ void NumericTrie::Node::insert_geopoint_helper(const uint64_t& cell_id, const ui
}
// Root node contains all the sequence ids present in the tree.
- if (!seq_ids.contains(seq_id)) {
- seq_ids.append(seq_id);
- }
+ ids_t::upsert(seq_ids, seq_id);
if (++level <= max_level) {
if (children == nullptr) {
@@ -562,12 +576,7 @@ void NumericTrie::Node::search_geopoints(const std::vector<uint64_t>& cell_ids,
}
for (auto const& match: matches) {
- auto const& m_seq_ids = match->seq_ids.uncompress();
- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {
- geo_result_ids.push_back(m_seq_ids[i]);
- }
-
- delete [] m_seq_ids;
+ ids_t::uncompress(match->seq_ids, geo_result_ids);
}
gfx::timsort(geo_result_ids.begin(), geo_result_ids.end());
@@ -580,7 +589,7 @@ void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, co
auto index = get_geopoint_index(cell_id, level);
while (level < max_level) {
- root->seq_ids.remove_value(id);
+ ids_t::erase(root->seq_ids, id);
if (root->children == nullptr || root->children[index] == nullptr) {
return;
@@ -590,12 +599,12 @@ void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, co
index = get_geopoint_index(cell_id, ++level);
}
- root->seq_ids.remove_value(id);
+ ids_t::erase(root->seq_ids, id);
if (root->children != nullptr && root->children[index] != nullptr) {
auto& child = root->children[index];
- child->seq_ids.remove_value(id);
- if (child->seq_ids.getLength() == 0) {
+ ids_t::erase(child->seq_ids, id);
+ if (ids_t::num_ids(child->seq_ids) == 0) {
delete child;
child = nullptr;
}
@@ -603,8 +612,8 @@ void NumericTrie::Node::delete_geopoint(const uint64_t& cell_id, uint32_t id, co
}
void NumericTrie::Node::get_all_ids(uint32_t*& ids, uint32_t& ids_length) {
- ids = seq_ids.uncompress();
- ids_length = seq_ids.getLength();
+ ids = ids_t::uncompress(seq_ids);
+ ids_length = ids_t::num_ids(seq_ids);
}
void NumericTrie::Node::search_less_than(const int64_t& value, const char& max_level,
@@ -620,12 +629,7 @@ void NumericTrie::Node::search_less_than(const int64_t& value, const char& max_l
std::vector<uint32_t> consolidated_ids;
for (auto const& match: matches) {
- auto const& m_seq_ids = match->seq_ids.uncompress();
- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {
- consolidated_ids.push_back(m_seq_ids[i]);
- }
-
- delete [] m_seq_ids;
+ ids_t::uncompress(match->seq_ids, consolidated_ids);
}
gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());
@@ -673,17 +677,12 @@ void NumericTrie::Node::search_range(const int64_t& low, const int64_t& high, co
return;
}
std::vector<NumericTrie::Node*> matches;
- search_range_helper(low, high >= indexable_limit(max_level) ? (int64_t) indexable_limit(max_level) - 1 : high,
+ search_range_helper(low, high >= indexable_limit(max_level) ? indexable_limit(max_level) : high,
max_level, matches);
std::vector<uint32_t> consolidated_ids;
for (auto const& match: matches) {
- auto const& m_seq_ids = match->seq_ids.uncompress();
- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {
- consolidated_ids.push_back(m_seq_ids[i]);
- }
-
- delete [] m_seq_ids;
+ ids_t::uncompress(match->seq_ids, consolidated_ids);
}
gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());
@@ -768,12 +767,7 @@ void NumericTrie::Node::search_greater_than(const int64_t& value, const char& ma
std::vector<uint32_t> consolidated_ids;
for (auto const& match: matches) {
- auto const& m_seq_ids = match->seq_ids.uncompress();
- for (uint32_t i = 0; i < match->seq_ids.getLength(); i++) {
- consolidated_ids.push_back(m_seq_ids[i]);
- }
-
- delete [] m_seq_ids;
+ ids_t::uncompress(match->seq_ids, consolidated_ids);
}
gfx::timsort(consolidated_ids.begin(), consolidated_ids.end());
@@ -817,7 +811,7 @@ void NumericTrie::Node::search_greater_than_helper(const int64_t& value, char& l
void NumericTrie::Node::search_equal_to(const int64_t& value, const char& max_level,
uint32_t*& ids, uint32_t& ids_length) {
- if (value >= indexable_limit(max_level)) {
+ if (value > indexable_limit(max_level)) {
return;
}
diff --git a/test/numeric_range_trie_test.cpp b/test/numeric_range_trie_test.cpp
index a53347ad..92bf01cc 100644
--- a/test/numeric_range_trie_test.cpp
+++ b/test/numeric_range_trie_test.cpp
@@ -1,7 +1,7 @@
#include <gtest/gtest.h>
#include <collection_manager.h>
#include "collection.h"
-#include "numeric_range_trie_test.h"
+#include "numeric_range_trie.h"
class NumericRangeTrieTest : public ::testing::Test {
protected: