Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Implement block based ID lists.
Kishore Nallan805932c7d760
diff --git a/include/id_list.h b/include/id_list.hnew file mode 100644index 00000000..65c2387f--- /dev/null+++ b/include/id_list.h@@ -0,0 +1,196 @@+#pragma once++#include <map>+#include <unordered_map>+#include "sorted_array.h"++typedef uint32_t last_id_t;++#define FOR_ELE_SIZE sizeof(uint32_t)+#define METADATA_OVERHEAD 5++/*+ Compressed chain of blocks that store the document IDs and offsets of a given token.+ Offsets of singular and multi-valued fields are encoded differently.+*/+class id_list_t {+public:++ // A block stores a sorted list of Document IDs compactly+ struct block_t {+ sorted_array ids;++ // link to next block+ block_t* next = nullptr;++ bool contains(uint32_t id);++ uint32_t upsert(uint32_t id);++ uint32_t erase(uint32_t id);++ uint32_t size() {+ return ids.getLength();+ }+ };++ class iterator_t {+ private:+ block_t* curr_block;+ uint32_t curr_index;++ block_t* end_block;++ public:+ // uncompressed data structure for performance+ uint32_t* ids = nullptr;++ explicit iterator_t(block_t* start, block_t* end);+ iterator_t(iterator_t&& rhs) noexcept;+ ~iterator_t();+ [[nodiscard]] bool valid() const;+ void next();+ void skip_to(uint32_t id);+ [[nodiscard]] uint32_t id() const;+ [[nodiscard]] inline uint32_t index() const;+ [[nodiscard]] inline block_t* block() const;+ };++ struct result_iter_state_t {+ const uint32_t* excluded_result_ids = nullptr;+ const size_t excluded_result_ids_size = 0;++ const uint32_t* filter_ids = nullptr;+ const size_t filter_ids_length = 0;++ size_t excluded_result_ids_index = 0;+ size_t filter_ids_index = 0;+ size_t index = 0;++ result_iter_state_t() = default;++ result_iter_state_t(uint32_t* excluded_result_ids, size_t excluded_result_ids_size,+ const uint32_t* filter_ids, const size_t filter_ids_length) : excluded_result_ids(excluded_result_ids),+ excluded_result_ids_size(excluded_result_ids_size),+ filter_ids(filter_ids), filter_ids_length(filter_ids_length) {}+ };++private:++ // maximum number of IDs (and associated offsets) to store in each block before another block is created+ const uint16_t BLOCK_MAX_ELEMENTS;+ uint32_t ids_length = 0;++ block_t root_block;++ // keeps track of the *last* ID in each block and is used for partial random access+ // e.g. 0..[9], 10..[19], 20..[29]+ // MUST be ordered+ std::map<last_id_t, block_t*> id_block_map;++ static bool at_end(const std::vector<id_list_t::iterator_t>& its);+ static bool at_end2(const std::vector<id_list_t::iterator_t>& its);++ static bool equals(std::vector<id_list_t::iterator_t>& its);+ static bool equals2(std::vector<id_list_t::iterator_t>& its);++ static void advance_all(std::vector<id_list_t::iterator_t>& its);+ static void advance_all2(std::vector<id_list_t::iterator_t>& its);++ static void advance_non_largest(std::vector<id_list_t::iterator_t>& its);+ static void advance_non_largest2(std::vector<id_list_t::iterator_t>& its);++ static uint32_t advance_smallest(std::vector<id_list_t::iterator_t>& its);+ static uint32_t advance_smallest2(std::vector<id_list_t::iterator_t>& its);++public:++ explicit id_list_t(uint16_t max_block_elements);++ ~id_list_t();++ static void split_block(block_t* src_block, block_t* dst_block);++ static void merge_adjacent_blocks(block_t* block1, block_t* block2, size_t num_block2_ids_to_move);++ void upsert(uint32_t id);++ void erase(uint32_t id);++ block_t* get_root();++ size_t num_blocks() const;++ size_t num_ids() const;++ uint32_t first_id();++ block_t* block_of(uint32_t id);++ bool contains(uint32_t id);++ bool contains_atleast_one(const uint32_t* target_ids, size_t target_ids_size);++ iterator_t new_iterator(block_t* start_block = nullptr, block_t* end_block = nullptr);++ static void merge(const std::vector<id_list_t*>& id_lists, std::vector<uint32_t>& result_ids);++ static void intersect(const std::vector<id_list_t*>& id_lists, std::vector<uint32_t>& result_ids);++ static bool take_id(result_iter_state_t& istate, uint32_t id);++ template<class T>+ static bool block_intersect(+ std::vector<id_list_t::iterator_t>& its,+ result_iter_state_t& istate,+ T func+ );++ uint32_t* uncompress();+};++template<class T>+bool id_list_t::block_intersect(std::vector<id_list_t::iterator_t>& its, result_iter_state_t& istate, T func) {++ switch (its.size()) {+ case 0:+ break;+ case 1:+ while(its[0].valid()) {+ if(id_list_t::take_id(istate, its[0].id())) {+ func(its[0].id(), its, istate.index);+ }++ its[0].next();+ }+ break;+ case 2:+ while(!at_end2(its)) {+ if(equals2(its)) {+ if(id_list_t::take_id(istate, its[0].id())) {+ func(its[0].id(), its, istate.index);+ }++ advance_all2(its);+ } else {+ advance_non_largest2(its);+ }+ }+ break;+ default:+ while(!at_end(its)) {+ if(equals(its)) {+ //LOG(INFO) << its[0].id();+ if(id_list_t::take_id(istate, its[0].id())) {+ func(its[0].id(), its, istate.index);+ }++ advance_all(its);+ } else {+ advance_non_largest(its);+ }+ }+ }++ return false;+}\ No newline at end of filediff --git a/include/ids_t.h b/include/ids_t.hnew file mode 100644index 00000000..7ebde632--- /dev/null+++ b/include/ids_t.h@@ -0,0 +1,162 @@+#pragma once++#include <cstdint>+#include <vector>+#include "id_list.h"+#include "threadpool.h"++#define IS_COMPACT_IDS(x) (((uintptr_t)(x) & 1))+#define SET_COMPACT_IDS(x) ((void*)((uintptr_t)(x) | 1))+#define RAW_IDS_PTR(x) ((void*)((uintptr_t)(x) & ~1))+#define COMPACT_IDS_PTR(x) ((compact_id_list_t*)((uintptr_t)(x) & ~1))++struct compact_id_list_t {+ // structured to get 4 byte alignment for `ids`+ uint8_t length = 0;+ uint16_t capacity = 0;++ // format: id1, id2,...+ uint32_t ids[];++ static compact_id_list_t* create(uint32_t num_ids, const std::vector<uint32_t>& ids);++ static compact_id_list_t* create(uint32_t num_ids, const uint32_t* ids);++ [[nodiscard]] id_list_t* to_full_ids_list() const;++ bool contains(uint32_t id);++ int64_t upsert(uint32_t id);++ void erase(uint32_t id);++ uint32_t first_id();+ uint32_t last_id();++ [[nodiscard]] uint32_t num_ids() const;++ bool contains_atleast_one(const uint32_t* target_ids, size_t target_ids_size);+};++class ids_t {+private:++ static void to_expanded_id_lists(const std::vector<void*>& raw_id_lists, std::vector<id_list_t*>& id_lists,+ std::vector<id_list_t*>& expanded_id_lists);++public:+ static constexpr size_t COMPACT_LIST_THRESHOLD_LENGTH = 64;+ static constexpr size_t MAX_BLOCK_ELEMENTS = 256;++ struct block_intersector_t {+ std::vector<id_list_t*> id_lists;+ std::vector<id_list_t*> expanded_id_lists;+ id_list_t::result_iter_state_t& iter_state;+ ThreadPool* thread_pool;+ size_t parallelize_min_ids;++ block_intersector_t(const std::vector<void*>& raw_id_lists,+ id_list_t::result_iter_state_t& iter_state,+ ThreadPool* thread_pool,+ size_t parallelize_min_ids = 1):+ iter_state(iter_state), thread_pool(thread_pool),+ parallelize_min_ids(parallelize_min_ids) {++ to_expanded_id_lists(raw_id_lists, id_lists, expanded_id_lists);++ if(id_lists.size() > 1) {+ std::sort(this->id_lists.begin(), this->id_lists.end(), [](id_list_t* a, id_list_t* b) {+ return a->num_blocks() < b->num_blocks();+ });+ }+ }++ ~block_intersector_t() {+ for(auto expanded_id_list: expanded_id_lists) {+ delete expanded_id_list;+ }+ }++ template<class T>+ bool intersect(T func, size_t concurrency=4);++ void split_lists(size_t concurrency, std::vector<std::vector<id_list_t::iterator_t>>& partial_its_vec);+ };++ static void upsert(void*& obj, uint32_t id);++ static void erase(void*& obj, uint32_t id);++ static void destroy_list(void*& obj);++ static uint32_t num_ids(const void* obj);++ static uint32_t first_id(const void* obj);++ static bool contains(const void* obj, uint32_t id);++ static bool contains_atleast_one(const void* obj, const uint32_t* target_ids, size_t target_ids_size);++ static void merge(const std::vector<void*>& id_lists, std::vector<uint32_t>& result_ids);++ static void intersect(const std::vector<void*>& id_lists, std::vector<uint32_t>& result_ids);++ static uint32_t* uncompress(void*& obj);+};++template<class T>+bool ids_t::block_intersector_t::intersect(T func, size_t concurrency) {+ // Split id lists into N chunks and intersect them in-parallel+ // 1. Sort id lists by number of blocks+ // 2. Iterate on the id list with least number of blocks on N-block windows+ // 3. On each window, pick the last block's last ID and identify blocks from other lists containing that ID+ // 4. Construct N groups of iterators this way (the last block must overlap on both sides of the window)++ if(id_lists.empty()) {+ return true;+ }++ if(id_lists[0]->num_ids() < parallelize_min_ids) {+ std::vector<id_list_t::iterator_t> its;+ its.reserve(id_lists.size());++ for(const auto& id_list: id_lists) {+ its.push_back(id_list->new_iterator());+ }++ id_list_t::block_intersect<T>(its, iter_state, func);+ return true;+ }++ std::vector<std::vector<id_list_t::iterator_t>> partial_its_vec(concurrency);+ split_lists(concurrency, partial_its_vec);++ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;+ size_t num_non_empty = 0;++ for(size_t i = 0; i < partial_its_vec.size(); i++) {+ auto& partial_its = partial_its_vec[i];++ if(partial_its.empty()) {+ continue;+ }++ num_non_empty++;++ thread_pool->enqueue([this, i, &func, &partial_its, &num_processed, &m_process, &cv_process]() {+ auto iter_state_copy = iter_state;+ iter_state_copy.index = i;+ id_list_t::block_intersect<T>(partial_its, iter_state_copy, func);+ std::unique_lock<std::mutex> lock(m_process);+ num_processed++;+ cv_process.notify_one();+ });+ }++ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){ return num_processed == num_non_empty; });++ return true;+}diff --git a/include/index.h b/include/index.hindex e4451b36..6de03cb5 100644--- a/include/index.h+++ b/include/index.h@@ -23,6 +23,7 @@#include "threadpool.h"#include "adi_tree.h"#include "tsl/htrie_set.h"+#include "id_list.h"static constexpr size_t ARRAY_FACET_DIM = 4;using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;@@ -455,7 +456,7 @@ private:spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;// this is used for wildcard queries- sorted_array seq_ids;+ id_list_t* seq_ids;std::vector<char> symbols_to_index;diff --git a/include/num_tree.h b/include/num_tree.hindex 39276cf0..f26b72ba 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -5,18 +5,15 @@#include "sorted_array.h"#include "array_utils.h"#include "art.h"+#include "ids_t.h"class num_tree_t {private:- std::map<int64_t, sorted_array*> int64map;+ std::map<int64_t, void*> int64map;public:- ~num_tree_t() {- for(auto& kv: int64map) {- delete kv.second;- }- }+ ~num_tree_t();void insert(int64_t value, uint32_t id);diff --git a/src/id_list.cpp b/src/id_list.cppnew file mode 100644index 00000000..908e06cf--- /dev/null+++ b/src/id_list.cpp@@ -0,0 +1,599 @@+#include "id_list.h"+#include <algorithm>+#include "for.h"++/* block_t operations */++bool id_list_t::block_t::contains(uint32_t id) {+ return ids.contains(id);+}++uint32_t id_list_t::block_t::upsert(const uint32_t id) {+ if (ids.contains(id)) {+ return 0;+ }++ ids.append(id);+ return 1;+}++uint32_t id_list_t::block_t::erase(const uint32_t id) {+ uint32_t doc_index = ids.indexOf(id);++ if (doc_index == ids.getLength()) {+ return 0;+ }++ ids.remove_value(id);+ return 1;+}++/* iterator_t operations */++id_list_t::iterator_t::iterator_t(id_list_t::block_t* start, id_list_t::block_t* end):+ curr_block(start), curr_index(0), end_block(end) {++ if(curr_block != end_block) {+ ids = curr_block->ids.uncompress();+ }+}++bool id_list_t::iterator_t::valid() const {+ return (curr_block != end_block) && (curr_index < curr_block->size());+}++void id_list_t::iterator_t::next() {+ curr_index++;+ if(curr_index == curr_block->size()) {+ curr_index = 0;+ curr_block = curr_block->next;++ delete [] ids;+ ids = nullptr;++ if(curr_block != end_block) {+ ids = curr_block->ids.uncompress();+ }+ }+}++uint32_t id_list_t::iterator_t::id() const {+ return ids[curr_index];+}++uint32_t id_list_t::iterator_t::index() const {+ return curr_index;+}++id_list_t::block_t* id_list_t::iterator_t::block() const {+ return curr_block;+}++void id_list_t::iterator_t::skip_to(uint32_t id) {+ bool skipped_block = false;+ while(curr_block != end_block && curr_block->ids.last() < id) {+ curr_block = curr_block->next;+ delete [] ids;++ ids = nullptr;++ if(curr_block != end_block) {+ ids = curr_block->ids.uncompress();+ }++ skipped_block = true;+ }++ if(skipped_block) {+ curr_index = 0;+ }++ while(curr_block != end_block && curr_index < curr_block->size() && this->id() < id) {+ curr_index++;+ }+}++id_list_t::iterator_t::~iterator_t() {+ delete [] ids;+ ids = nullptr;+}++id_list_t::iterator_t::iterator_t(iterator_t&& rhs) noexcept {+ curr_block = rhs.curr_block;+ curr_index = rhs.curr_index;+ end_block = rhs.end_block;+ ids = rhs.ids;++ rhs.curr_block = nullptr;+ rhs.end_block = nullptr;+ rhs.ids = nullptr;+}++/* id_list_t operations */++id_list_t::id_list_t(uint16_t max_block_elements): BLOCK_MAX_ELEMENTS(max_block_elements) {+ if(max_block_elements <= 1) {+ throw std::invalid_argument("max_block_elements must be > 1");+ }+}++id_list_t::~id_list_t() {+ block_t* block = root_block.next;+ while(block != nullptr) {+ block_t* next_block = block->next;+ delete block;+ block = next_block;+ }+}++void id_list_t::merge_adjacent_blocks(id_list_t::block_t* block1, id_list_t::block_t* block2,+ size_t num_block2_ids_to_move) {+ // merge ids+ uint32_t* ids1 = block1->ids.uncompress();+ uint32_t* ids2 = block2->ids.uncompress();++ uint32_t* new_ids = new uint32_t[block1->size() + num_block2_ids_to_move];+ std::memmove(new_ids, ids1, sizeof(uint32_t) * block1->size());+ std::memmove(new_ids + block1->size(), ids2, sizeof(uint32_t) * num_block2_ids_to_move);++ block1->ids.load(new_ids, block1->size() + num_block2_ids_to_move);+ if(block2->size() != num_block2_ids_to_move) {+ block2->ids.load(ids2 + num_block2_ids_to_move, block2->size() - num_block2_ids_to_move);+ } else {+ block2->ids.load(nullptr, 0);+ }++ delete [] ids1;+ delete [] ids2;+ delete [] new_ids;+}++void id_list_t::split_block(id_list_t::block_t* src_block, id_list_t::block_t* dst_block) {+ if(src_block->size() <= 1) {+ return;+ }++ uint32_t* raw_ids = src_block->ids.uncompress();+ size_t ids_first_half_length = (src_block->size() / 2);+ size_t ids_second_half_length = (src_block->size() - ids_first_half_length);+ src_block->ids.load(raw_ids, ids_first_half_length);+ dst_block->ids.load(raw_ids + ids_first_half_length, ids_second_half_length);++ delete [] raw_ids;+}++void id_list_t::upsert(const uint32_t id) {+ // first we will locate the block where `id` should reside+ block_t* upsert_block;+ last_id_t before_upsert_last_id;++ if(id_block_map.empty()) {+ upsert_block = &root_block;+ before_upsert_last_id = UINT32_MAX;+ } else {+ const auto it = id_block_map.lower_bound(id);+ upsert_block = (it == id_block_map.end()) ? id_block_map.rbegin()->second : it->second;+ before_upsert_last_id = upsert_block->ids.last();+ }++ // happy path: upsert_block is not full+ if(upsert_block->size() < BLOCK_MAX_ELEMENTS) {+ uint32_t num_inserted = upsert_block->upsert(id);+ ids_length += num_inserted;++ last_id_t after_upsert_last_id = upsert_block->ids.last();+ if(before_upsert_last_id != after_upsert_last_id) {+ id_block_map.erase(before_upsert_last_id);+ id_block_map.emplace(after_upsert_last_id, upsert_block);+ }+ } else {+ block_t* new_block = new block_t;++ if(upsert_block->next == nullptr && upsert_block->ids.last() < id) {+ // appending to the end of the last block where the id will reside on a newly block+ uint32_t num_inserted = new_block->upsert(id);+ ids_length += num_inserted;+ } else {+ // upsert and then split block+ uint32_t num_inserted = upsert_block->upsert(id);+ ids_length += num_inserted;++ // evenly divide elements between both blocks+ split_block(upsert_block, new_block);++ last_id_t after_upsert_last_id = upsert_block->ids.last();+ id_block_map.erase(before_upsert_last_id);+ id_block_map.emplace(after_upsert_last_id, upsert_block);+ }++ last_id_t after_new_block_id = new_block->ids.last();+ id_block_map.emplace(after_new_block_id, new_block);++ new_block->next = upsert_block->next;+ upsert_block->next = new_block;+ }+}++void id_list_t::erase(const uint32_t id) {+ const auto it = id_block_map.lower_bound(id);++ if(it == id_block_map.end()) {+ return ;+ }++ block_t* erase_block = it->second;+ last_id_t before_last_id = it->first;+ uint32_t num_erased = erase_block->erase(id);+ ids_length -= num_erased;++ size_t new_ids_length = erase_block->size();++ if(new_ids_length == 0) {+ // happens when the last element of last block is deleted++ if(erase_block != &root_block) {+ // since we will be deleting the empty node, set the previous node's next pointer to null+ std::prev(it)->second->next = nullptr;+ delete erase_block;+ } else {+ // The root block cannot be empty if there are other blocks so we will pull some contents from next block+ // This is only an issue for blocks with max size of 2+ if(root_block.next != nullptr) {+ auto next_block_last_id = erase_block->next->ids.last();+ merge_adjacent_blocks(erase_block, erase_block->next, erase_block->next->size()/2);+ id_block_map.erase(next_block_last_id);++ id_block_map.emplace(erase_block->next->ids.last(), erase_block->next);+ id_block_map.emplace(erase_block->ids.last(), erase_block);+ }+ }++ id_block_map.erase(before_last_id);++ return;+ }++ if(new_ids_length >= BLOCK_MAX_ELEMENTS/2 || erase_block->next == nullptr) {+ last_id_t after_last_id = erase_block->ids.last();+ if(before_last_id != after_last_id) {+ id_block_map.erase(before_last_id);+ id_block_map.emplace(after_last_id, erase_block);+ }++ return ;+ }++ // block is less than 50% of max capacity and contains a next node which we can refill from++ auto next_block = erase_block->next;+ last_id_t next_block_last_id = next_block->ids.last();++ if(erase_block->size() + next_block->size() <= BLOCK_MAX_ELEMENTS) {+ // we can merge the contents of next block with `erase_block` and delete the next block+ merge_adjacent_blocks(erase_block, next_block, next_block->size());+ erase_block->next = next_block->next;+ delete next_block;++ id_block_map.erase(next_block_last_id);+ } else {+ // Only part of the next block can be moved over.+ // We will move only 50% of max elements to ensure that we don't end up "flipping" adjacent blocks:+ // 1, 5 -> 5, 1+ size_t num_block2_ids = BLOCK_MAX_ELEMENTS/2;+ merge_adjacent_blocks(erase_block, next_block, num_block2_ids);+ // NOTE: we don't have to update `id_block_map` for `next_block` as last element doesn't change+ }++ last_id_t after_last_id = erase_block->ids.last();+ if(before_last_id != after_last_id) {+ id_block_map.erase(before_last_id);+ id_block_map.emplace(after_last_id, erase_block);+ }+}++id_list_t::block_t* id_list_t::get_root() {+ return &root_block;+}++size_t id_list_t::num_blocks() const {+ return id_block_map.size();+}++uint32_t id_list_t::first_id() {+ if(ids_length == 0) {+ return 0;+ }++ return root_block.ids.at(0);+}++id_list_t::block_t* id_list_t::block_of(uint32_t id) {+ const auto it = id_block_map.lower_bound(id);+ if(it == id_block_map.end()) {+ return nullptr;+ }++ return it->second;+}++void id_list_t::merge(const std::vector<id_list_t*>& id_lists, std::vector<uint32_t>& result_ids) {+ auto its = std::vector<id_list_t::iterator_t>();+ its.reserve(id_lists.size());++ size_t sum_sizes = 0;++ for(const auto& id_list: id_lists) {+ its.push_back(id_list->new_iterator());+ sum_sizes += id_list->num_ids();+ }++ result_ids.reserve(sum_sizes);+ size_t num_lists = its.size();++ switch (num_lists) {+ case 2:+ while(!at_end2(its)) {+ if(equals2(its)) {+ //LOG(INFO) << its[0].id();+ result_ids.push_back(its[0].id());+ advance_all2(its);+ } else {+ uint32_t smallest_value = advance_smallest2(its);+ result_ids.push_back(smallest_value);+ }+ }++ while(its[0].valid()) {+ result_ids.push_back(its[0].id());+ its[0].next();+ }++ while(its[1].valid()) {+ result_ids.push_back(its[1].id());+ its[1].next();+ }++ break;+ default:+ while(!at_end(its)) {+ if(equals(its)) {+ result_ids.push_back(its[0].id());+ advance_all(its);+ } else {+ uint32_t smallest_value = advance_smallest(its);+ result_ids.push_back(smallest_value);+ }+ }++ for(auto& it: its) {+ while(it.valid()) {+ result_ids.push_back(it.id());+ it.next();+ }+ }+ }+}++// Inspired by: https://stackoverflow.com/a/25509185/131050+void id_list_t::intersect(const std::vector<id_list_t*>& id_lists, std::vector<uint32_t>& result_ids) {+ if(id_lists.empty()) {+ return;+ }++ if(id_lists.size() == 1) {+ result_ids.reserve(id_lists[0]->ids_length);+ auto it = id_lists[0]->new_iterator();+ while(it.valid()) {+ result_ids.push_back(it.id());+ it.next();+ }++ return ;+ }++ auto its = std::vector<id_list_t::iterator_t>();+ its.reserve(id_lists.size());++ for(const auto& id_list: id_lists) {+ its.push_back(id_list->new_iterator());+ }++ size_t num_lists = its.size();++ switch (num_lists) {+ case 2:+ while(!at_end2(its)) {+ if(equals2(its)) {+ //LOG(INFO) << its[0].id();+ result_ids.push_back(its[0].id());+ advance_all2(its);+ } else {+ advance_non_largest2(its);+ }+ }+ break;+ default:+ while(!at_end(its)) {+ if(equals(its)) {+ //LOG(INFO) << its[0].id();+ result_ids.push_back(its[0].id());+ advance_all(its);+ } else {+ advance_non_largest(its);+ }+ }+ }+}++bool id_list_t::at_end(const std::vector<id_list_t::iterator_t>& its) {+ // if any one iterator is at end, we can stop+ for(const auto& it : its) {+ if(!it.valid()) {+ return true;+ }+ }++ return false;+}++bool id_list_t::at_end2(const std::vector<id_list_t::iterator_t>& its) {+ // if any one iterator is at end, we can stop+ return !its[0].valid() || !its[1].valid();+}++bool id_list_t::equals(std::vector<id_list_t::iterator_t>& its) {+ for(size_t i = 0; i < its.size() - 1; i++) {+ if(its[i].id() != its[i+1].id()) {+ return false;+ }+ }++ return true;+}++bool id_list_t::equals2(std::vector<id_list_t::iterator_t>& its) {+ return its[0].id() == its[1].id();+}++id_list_t::iterator_t id_list_t::new_iterator(block_t* start_block, block_t* end_block) {+ start_block = (start_block == nullptr) ? &root_block : start_block;+ return id_list_t::iterator_t(start_block, end_block);+}++void id_list_t::advance_all(std::vector<id_list_t::iterator_t>& its) {+ for(auto& it: its) {+ it.next();+ }+}++void id_list_t::advance_all2(std::vector<id_list_t::iterator_t>& its) {+ its[0].next();+ its[1].next();+}++void id_list_t::advance_non_largest(std::vector<id_list_t::iterator_t>& its) {+ // we will find the iter with greatest value and then advance the rest until their value catches up+ uint32_t greatest_value = 0;++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() > greatest_value) {+ greatest_value = its[i].id();+ }+ }++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() != greatest_value) {+ its[i].skip_to(greatest_value);+ }+ }+}++void id_list_t::advance_non_largest2(std::vector<id_list_t::iterator_t>& its) {+ if(its[0].id() > its[1].id()) {+ its[1].skip_to(its[0].id());+ } else {+ its[0].skip_to(its[1].id());+ }+}++uint32_t id_list_t::advance_smallest(std::vector<id_list_t::iterator_t>& its) {+ // we will advance the iterator(s) with the smallest value and then return that value+ uint32_t smallest_value = UINT32_MAX;++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() < smallest_value) {+ smallest_value = its[i].id();+ }+ }++ for(size_t i = 0; i < its.size(); i++) {+ if(its[i].id() == smallest_value) {+ its[i].next();+ }+ }++ return smallest_value;+}++uint32_t id_list_t::advance_smallest2(std::vector<id_list_t::iterator_t>& its) {+ uint32_t smallest_value = 0;++ if(its[0].id() < its[1].id()) {+ smallest_value = its[0].id();+ its[0].next();+ } else {+ smallest_value = its[1].id();+ its[1].next();+ }++ return smallest_value;+}++size_t id_list_t::num_ids() const {+ return ids_length;+}++bool id_list_t::contains(uint32_t id) {+ const auto it = id_block_map.lower_bound(id);++ if(it == id_block_map.end()) {+ return false;+ }++ block_t* potential_block = it->second;+ return potential_block->contains(id);+}++bool id_list_t::contains_atleast_one(const uint32_t* target_ids, size_t target_ids_size) {+ id_list_t::iterator_t it = new_iterator();+ size_t target_ids_index = 0;++ while(target_ids_index < target_ids_size && it.valid()) {+ uint32_t id = it.id();++ if(id == target_ids[target_ids_index]) {+ return true;+ } else {+ // advance smallest value+ if(id > target_ids[target_ids_index]) {+ while(target_ids_index < target_ids_size && target_ids[target_ids_index] < id) {+ target_ids_index++;+ }+ } else {+ it.skip_to(target_ids[target_ids_index]);+ }+ }+ }++ return false;+}++bool id_list_t::take_id(result_iter_state_t& istate, uint32_t id) {+ // decide if this result id should be excluded+ if(istate.excluded_result_ids_size != 0) {+ if (std::binary_search(istate.excluded_result_ids,+ istate.excluded_result_ids + istate.excluded_result_ids_size, id)) {+ return false;+ }+ }++ // decide if this result be matched with filter results+ if(istate.filter_ids_length != 0) {+ return std::binary_search(istate.filter_ids, istate.filter_ids + istate.filter_ids_length, id);+ }++ return true;+}++uint32_t* id_list_t::uncompress() {+ uint32_t* arr = new uint32_t[ids_length];+ auto it = new_iterator();+ size_t i = 0;++ while(it.valid()) {+ arr[i++] = it.id();+ it.next();+ }++ return arr;+}diff --git a/src/ids_t.cpp b/src/ids_t.cppnew file mode 100644index 00000000..13636be5--- /dev/null+++ b/src/ids_t.cpp@@ -0,0 +1,425 @@+#include "ids_t.h"+#include "id_list.h"++int64_t compact_id_list_t::upsert(const uint32_t id) {+ // format: id1, id2, id3+ uint32_t last_id = (length == 0) ? 0 : ids[length - 1];+ int64_t extra_length_needed = 0;++ if(length == 0 || id > last_id) {+ extra_length_needed = 1;+ if(length + extra_length_needed > capacity) {+ // enough storage should have been provided upstream+ return (length + extra_length_needed) - capacity;+ }++ // can just append to the end+ ids[length++] = id;+ } else {+ // locate position and shift contents to make space available+ int64_t i = 0;++ while(i < length) {+ size_t existing_id = ids[i];+ if(existing_id == id) {+ break;+ }++ else if(existing_id > id) {+ extra_length_needed = 1;+ if(length + extra_length_needed > capacity) {+ // enough storage should have been provided upstream+ return (length + extra_length_needed) - capacity;+ }++ // shift index [i..length-1] by `extra_length_needed` positions+ int64_t shift_index = length + extra_length_needed - 1;+ while((shift_index - extra_length_needed) >= 0 && shift_index >= i) {+ ids[shift_index] = ids[shift_index - extra_length_needed];+ shift_index--;+ }+ // now store the new offsets in the shifted space+ ids[i++] = id;+ break;+ }++ i++;+ }++ length += extra_length_needed;+ }++ return 0;+}++void compact_id_list_t::erase(const uint32_t id) {+ // locate position and shift contents to collapse space vacated+ size_t i = 0;+ while(i < length) {+ size_t existing_id = ids[i];+ if(existing_id > id) {+ // not found!+ return ;+ }++ if(existing_id == id) {+ size_t shift_offset = 1;+ while(i+shift_offset < length) {+ ids[i] = ids[i + shift_offset];+ i++;+ }++ length -= shift_offset;+ break;+ }++ i++;+ }+}++compact_id_list_t* compact_id_list_t::create(uint32_t num_ids, const std::vector<uint32_t>& ids) {+ return create(num_ids, &ids[0]);+}++compact_id_list_t* compact_id_list_t::create(uint32_t num_ids, const uint32_t* ids) {+ // format: id1, id2, id3, ...++ compact_id_list_t* pl = (compact_id_list_t*) malloc(sizeof(compact_id_list_t) ++ (num_ids * sizeof(uint32_t)));++ pl->length = 0;+ pl->capacity = num_ids;++ for(size_t i = 0; i < num_ids; i++) {+ pl->upsert(ids[i]);+ }++ return pl;+}++id_list_t* compact_id_list_t::to_full_ids_list() const {+ id_list_t* pl = new id_list_t(ids_t::MAX_BLOCK_ELEMENTS);++ size_t i = 0;+ while(i < length) {+ size_t existing_id = ids[i];+ pl->upsert(existing_id);+ i++;+ }++ return pl;+}++uint32_t compact_id_list_t::last_id() {+ return (length == 0) ? UINT32_MAX : ids[length - 1];+}++uint32_t compact_id_list_t::num_ids() const {+ return length;+}++uint32_t compact_id_list_t::first_id() {+ if(length == 0) {+ return 0;+ }++ return ids[0];+}++bool compact_id_list_t::contains(uint32_t id) {+ size_t i = 0;+ while(i < length) {+ size_t existing_id = ids[i];+ if(existing_id > id) {+ // not found!+ return false;+ }++ if(existing_id == id) {+ return true;+ }++ i++;+ }++ return false;+}++bool compact_id_list_t::contains_atleast_one(const uint32_t* target_ids, size_t target_ids_size) {+ size_t i = 0;+ size_t target_ids_index = 0;++ while(i < length && target_ids_index < target_ids_size) {+ size_t num_existing_offsets = ids[i];+ size_t existing_id = ids[i + num_existing_offsets + 1];++ // Returns iterator to the first element that is >= to value or last if no such element is found.+ size_t found_index = std::lower_bound(target_ids + target_ids_index,+ target_ids + target_ids_size, existing_id) - target_ids;++ if(found_index == target_ids_size) {+ // all elements are lesser than lowest value (existing_id), so we can stop looking+ return false;+ } else {+ if(target_ids[found_index] == existing_id) {+ return true;+ }++ // adjust lower bound to found_index+1 whose value is >= `existing_id`+ target_ids_index = found_index;+ }++ i += num_existing_offsets + 2;+ }++ return false;+}++/* posting operations */++void ids_t::upsert(void*& obj, uint32_t id) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = (compact_id_list_t*) RAW_IDS_PTR(obj);+ int64_t extra_capacity_required = list->upsert(id);++ if(extra_capacity_required == 0) {+ // upsert succeeded+ return;+ }++ if((list->capacity + extra_capacity_required) > COMPACT_LIST_THRESHOLD_LENGTH) {+ // we have to convert to a full posting list+ id_list_t* full_list = list->to_full_ids_list();+ free(list);+ obj = full_list;+ }++ else {+ // grow the container by 30%+ size_t new_capacity = std::min<size_t>((list->capacity + extra_capacity_required) * 1.3,+ COMPACT_LIST_THRESHOLD_LENGTH);++ size_t new_capacity_bytes = sizeof(compact_id_list_t) + (new_capacity * sizeof(uint32_t));+ auto new_list = (compact_id_list_t *) realloc(list, new_capacity_bytes);+ if(new_list == nullptr) {+ abort();+ }++ list = new_list;+ list->capacity = new_capacity;+ obj = SET_COMPACT_IDS(list);++ list->upsert(id);++ return ;+ }+ }++ // either `obj` is already a full list or was converted to a full list above+ id_list_t* list = (id_list_t*)(obj);+ list->upsert(id);+}++void ids_t::erase(void*& obj, uint32_t id) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ list->erase(id);++ // if the list becomes too small, we resize it to save memory+ if(list->length < list->capacity/2) {+ // resize container+ size_t new_capacity = list->capacity/2;+ size_t new_capacity_bytes = sizeof(compact_id_list_t) + (new_capacity * sizeof(uint32_t));+ auto new_list = (compact_id_list_t *) realloc(list, new_capacity_bytes);+ if(new_list == nullptr) {+ abort();+ }++ list = new_list;+ list->capacity = new_capacity;+ obj = SET_COMPACT_IDS(list);+ }++ } else {+ id_list_t* list = (id_list_t*)(obj);+ list->erase(id);++ if(list->num_blocks() == 1 && list->get_root()->size() <= COMPACT_LIST_THRESHOLD_LENGTH) {+ // convert to compact posting format+ auto root_block = list->get_root();+ auto ids = root_block->ids.uncompress();++ compact_id_list_t* compact_list = compact_id_list_t::create(root_block->size(), ids);++ delete [] ids;+ delete list;++ obj = SET_COMPACT_IDS(compact_list);+ }+ }+}++uint32_t ids_t::num_ids(const void* obj) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ return list->num_ids();+ } else {+ id_list_t* list = (id_list_t*)(obj);+ return list->num_ids();+ }+}++uint32_t ids_t::first_id(const void* obj) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ return list->first_id();+ } else {+ id_list_t* list = (id_list_t*)(obj);+ return list->first_id();+ }+}++bool ids_t::contains(const void* obj, uint32_t id) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ return list->contains(id);+ } else {+ id_list_t* list = (id_list_t*)(obj);+ return list->contains(id);+ }+}++bool ids_t::contains_atleast_one(const void* obj, const uint32_t* target_ids, size_t target_ids_size) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ return list->contains_atleast_one(target_ids, target_ids_size);+ } else {+ id_list_t* list = (id_list_t*)(obj);+ return list->contains_atleast_one(target_ids, target_ids_size);+ }+}++void ids_t::merge(const std::vector<void*>& raw_posting_lists, std::vector<uint32_t>& result_ids) {+ // we will have to convert the compact posting list (if any) to full form+ std::vector<id_list_t*> id_lists;+ std::vector<id_list_t*> expanded_id_lists;+ to_expanded_id_lists(raw_posting_lists, id_lists, expanded_id_lists);++ id_list_t::merge(id_lists, result_ids);++ for(id_list_t* expanded_plist: expanded_id_lists) {+ delete expanded_plist;+ }+}++void ids_t::intersect(const std::vector<void*>& raw_posting_lists, std::vector<uint32_t>& result_ids) {+ // we will have to convert the compact posting list (if any) to full form+ std::vector<id_list_t*> id_lists;+ std::vector<id_list_t*> expanded_id_lists;+ to_expanded_id_lists(raw_posting_lists, id_lists, expanded_id_lists);++ id_list_t::intersect(id_lists, result_ids);++ for(auto expanded_plist: expanded_id_lists) {+ delete expanded_plist;+ }+}++void ids_t::to_expanded_id_lists(const std::vector<void*>& raw_posting_lists, std::vector<id_list_t*>& id_lists,+ std::vector<id_list_t*>& expanded_id_lists) {+ for(size_t i = 0; i < raw_posting_lists.size(); i++) {+ auto raw_posting_list = raw_posting_lists[i];++ if(IS_COMPACT_IDS(raw_posting_list)) {+ auto compact_posting_list = COMPACT_IDS_PTR(raw_posting_list);+ id_list_t* full_posting_list = compact_posting_list->to_full_ids_list();+ id_lists.emplace_back(full_posting_list);+ expanded_id_lists.push_back(full_posting_list);+ } else {+ id_list_t* full_posting_list = (id_list_t*)(raw_posting_list);+ id_lists.emplace_back(full_posting_list);+ }+ }+}++void ids_t::destroy_list(void*& obj) {+ if(obj == nullptr) {+ return;+ }++ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ free(list); // assigned via malloc, so must be free()d+ } else {+ id_list_t* list = (id_list_t*)(obj);+ delete list;+ }++ obj = nullptr;+}++uint32_t* ids_t::uncompress(void*& obj) {+ if(IS_COMPACT_IDS(obj)) {+ compact_id_list_t* list = COMPACT_IDS_PTR(obj);+ uint32_t* arr = new uint32_t[list->length];+ std::memcpy(arr, list->ids, list->length * sizeof(uint32_t));+ return arr;+ } else {+ id_list_t* list = (id_list_t*)(obj);+ return list->uncompress();+ }+}++void ids_t::block_intersector_t::split_lists(size_t concurrency,+ std::vector<std::vector<id_list_t::iterator_t>>& partial_its_vec) {+ const size_t num_blocks = this->id_lists[0]->num_blocks();+ const size_t window_size = (num_blocks + concurrency - 1) / concurrency; // rounds up++ size_t blocks_traversed = 0;+ id_list_t::block_t* start_block = this->id_lists[0]->get_root();+ id_list_t::block_t* curr_block = start_block;++ size_t window_index = 0;++ while(curr_block != nullptr) {+ blocks_traversed++;+ if(blocks_traversed % window_size == 0 || blocks_traversed == num_blocks) {+ // construct partial iterators and intersect within them++ std::vector<id_list_t::iterator_t>& partial_its = partial_its_vec[window_index];++ for(size_t i = 0; i < this->id_lists.size(); i++) {+ id_list_t::block_t* p_start_block = nullptr;+ id_list_t::block_t* p_end_block = nullptr;++ // [1, 2] [3, 4] [5, 6]+ // [3, 5] [6]++ if(i == 0) {+ p_start_block = start_block;+ p_end_block = curr_block->next;+ } else {+ auto start_block_first_id = start_block->ids.at(0);+ auto end_block_last_id = curr_block->ids.last();++ p_start_block = this->id_lists[i]->block_of(start_block_first_id);+ id_list_t::block_t* last_block = this->id_lists[i]->block_of(end_block_last_id);++ if(p_start_block == last_block && p_start_block != nullptr) {+ p_end_block = p_start_block->next;+ } else {+ p_end_block = last_block == nullptr ? nullptr : last_block->next;+ }+ }++ partial_its.emplace_back(p_start_block, p_end_block);+ }++ start_block = curr_block->next;+ window_index++;+ }++ curr_block = curr_block->next;+ }+}+diff --git a/src/index.cpp b/src/index.cppindex 770168d9..7f2baea8 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -44,7 +44,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):name(name), collection_id(collection_id), store(store), thread_pool(thread_pool),search_schema(search_schema), facet_schema(facet_schema), sort_schema(sort_schema),- symbols_to_index(symbols_to_index), token_separators(token_separators) {+ seq_ids(new id_list_t(256)), symbols_to_index(symbols_to_index), token_separators(token_separators) {for(const auto & fname_field: search_schema) {if(fname_field.second.is_string()) {@@ -173,6 +173,8 @@ Index::~Index() {}facet_index_v3.clear();++ delete seq_ids;}int64_t Index::get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field) {@@ -625,7 +627,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}if(!record.is_update && record.indexed.ok()) {// for updates, the seq_id will already exist- seq_ids.append(record.seq_id);+ seq_ids->upsert(record.seq_id);}}@@ -1362,8 +1364,8 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,size_t to_exclude_ids_len = 0;num_tree->search(EQUALS, bool_int64, &to_exclude_ids, to_exclude_ids_len);- auto all_ids = seq_ids.uncompress();- auto all_ids_size = seq_ids.getLength();+ auto all_ids = seq_ids->uncompress();+ auto all_ids_size = seq_ids->num_ids();uint32_t* excluded_ids = nullptr;size_t excluded_ids_len = 0;@@ -1568,18 +1570,19 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,if(a_filter.comparators[0] == NOT_EQUALS) {// exclude records from existing IDs (from previous filters or ALL records)// upstream will guarantee that NOT_EQUALS is placed right at the end of filters list+ uint32_t* excluded_strt_ids = nullptr;+ size_t excluded_strt_size = 0;+if(ids == nullptr) {if(filter_ids == nullptr) {- ids = seq_ids.uncompress();- ids_size = seq_ids.getLength();+ ids = seq_ids->uncompress();+ ids_size = seq_ids->num_ids();} else {ids = filter_ids;ids_size = filter_ids_length;}}- uint32_t* excluded_strt_ids = nullptr;- size_t excluded_strt_size = 0;excluded_strt_size = ArrayUtils::exclude_scalar(ids, ids_size, strt_ids,strt_ids_size, &excluded_strt_ids);@@ -3012,8 +3015,8 @@ void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::s// if filtered results are not available, use the seq_ids index to generate the list of all document idsif(filters.empty() && filter_ids_length == 0) {- filter_ids_length = seq_ids.getLength();- filter_ids = seq_ids.uncompress();+ filter_ids_length = seq_ids->num_ids();+ filter_ids = seq_ids->uncompress();}if(!curated_ids.empty()) {@@ -3829,7 +3832,7 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc}if(!is_update) {- seq_ids.remove_value(seq_id);+ seq_ids->erase(seq_id);}return Option<uint32_t>(seq_id);@@ -4370,7 +4373,7 @@ void Index::scrub_reindex_doc(const std::unordered_map<std::string, field>& searsize_t Index::num_seq_ids() const {std::shared_lock lock(mutex);- return seq_ids.getLength();+ return seq_ids->num_ids();}void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const string& field_name,diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 5d4fdc31..d0b89dc2 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -3,11 +3,13 @@void num_tree_t::insert(int64_t value, uint32_t id) {if (int64map.count(value) == 0) {- int64map.emplace(value, new sorted_array);- }-- if (!int64map[value]->contains(id)) {- int64map[value]->append(id);+ int64map.emplace(value, SET_COMPACT_IDS(compact_id_list_t::create(1, {id})));+ } else {+ auto ids = int64map[value];+ if (!ids_t::contains(ids, id)) {+ ids_t::upsert(ids, id);+ int64map[value] = ids;+ }}}@@ -20,9 +22,9 @@ void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** istd::vector<uint32_t> consolidated_ids;while(it_start != int64map.end() && it_start->first <= end) {- uint32_t* values = it_start->second->uncompress();+ uint32_t* values = ids_t::uncompress(it_start->second);- for(size_t i = 0; i < it_start->second->getLength(); i++) {+ for(size_t i = 0; i < ids_t::num_ids(it_start->second); i++) {consolidated_ids.push_back(values[i]);}@@ -50,14 +52,14 @@ size_t num_tree_t::get(int64_t value, std::vector<uint32_t>& geo_result_ids) {return 0;}- uint32_t* ids = it->second->uncompress();- for(size_t i = 0; i < it->second->getLength(); i++) {+ uint32_t* ids = ids_t::uncompress(it->second);+ for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {geo_result_ids.push_back(ids[i]);}delete [] ids;- return it->second->getLength();+ return ids_t::num_ids(it->second);}void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids, size_t& ids_len) {@@ -69,8 +71,8 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** idsconst auto& it = int64map.find(value);if(it != int64map.end()) {uint32_t *out = nullptr;- uint32_t* val_ids = it->second->uncompress();- ids_len = ArrayUtils::or_scalar(val_ids, it->second->getLength(),+ uint32_t* val_ids = ids_t::uncompress(it->second);+ ids_len = ArrayUtils::or_scalar(val_ids, ids_t::num_ids(it->second),*ids, ids_len, &out);delete[] *ids;*ids = out;@@ -90,9 +92,9 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** idsstd::vector<uint32_t> consolidated_ids;while(iter_ge_value != int64map.end()) {- uint32_t* values = iter_ge_value->second->uncompress();+ uint32_t* values = ids_t::uncompress(iter_ge_value->second);- for(size_t i = 0; i < iter_ge_value->second->getLength(); i++) {+ for(size_t i = 0; i < ids_t::num_ids(iter_ge_value->second); i++) {consolidated_ids.push_back(values[i]);}@@ -123,9 +125,9 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** idsauto it = int64map.begin();while(it != iter_ge_value) {- uint32_t* values = it->second->uncompress();+ uint32_t* values = ids_t::uncompress(it->second);- for(size_t i = 0; i < it->second->getLength(); i++) {+ for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {consolidated_ids.push_back(values[i]);}@@ -135,8 +137,8 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** ids// for LESS_THAN_EQUALS, check if last iter entry is equal to valueif(it != int64map.end() && comparator == LESS_THAN_EQUALS && it->first == value) {- uint32_t* values = it->second->uncompress();- for(size_t i = 0; i < it->second->getLength(); i++) {+ uint32_t* values = ids_t::uncompress(it->second);+ for(size_t i = 0; i < ids_t::num_ids(it->second); i++) {consolidated_ids.push_back(values[i]);}delete [] values;@@ -161,15 +163,24 @@ void num_tree_t::search(NUM_COMPARATOR comparator, int64_t value, uint32_t** idsvoid num_tree_t::remove(uint64_t value, uint32_t id) {if(int64map.count(value) != 0) {- sorted_array* arr = int64map[value];- arr->remove_value(id);- if(arr->getLength() == 0) {- delete arr;+ void* arr = int64map[value];+ ids_t::erase(arr, id);++ if(ids_t::num_ids(arr) == 0) {+ ids_t::destroy_list(arr);int64map.erase(value);+ } else {+ int64map[value] = arr;}}}size_t num_tree_t::size() {return int64map.size();-}\ No newline at end of file+}++num_tree_t::~num_tree_t() {+ for(auto& kv: int64map) {+ ids_t::destroy_list(kv.second);+ }+}diff --git a/test/num_tree_test.cpp b/test/num_tree_test.cppindex 1a278ffc..b44f5aa5 100644--- a/test/num_tree_test.cpp+++ b/test/num_tree_test.cpp@@ -42,3 +42,33 @@ TEST(NumTreeTest, Searches) {delete [] ids;ids = nullptr;}++TEST(NumTreeTest, EraseFullList) {+ num_tree_t tree;++ // this stores the IDs as a full list+ for(size_t i = 0; i < 200; i++) {+ tree.insert(0, i);+ }++ // we erase all but 1 ID+ for(size_t i = 0; i < 199; i++) {+ tree.remove(0, i);+ }++ // now try searching for the value+ uint32_t* ids = nullptr;+ size_t ids_len;++ tree.search(NUM_COMPARATOR::EQUALS, 0, &ids, ids_len);+ ASSERT_EQ(1, ids_len);+ ASSERT_EQ(199, ids[0]);+ delete [] ids;+ ids = nullptr;++ // deleting the last ID as well+ tree.remove(0, 199);++ tree.search(NUM_COMPARATOR::EQUALS, 0, &ids, ids_len);+ ASSERT_EQ(nullptr, ids);+}