Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

using numerical index for new data structure

krunal131358e2de03ce92
diff --git a/include/facet_index.h b/include/facet_index.h
index 6cf0bb84..cafb6c74 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -43,7 +43,7 @@ public:
~facet_index_t();
- uint32_t insert(const std::string& field, const std::string& value, uint32_t id);
+ uint32_t insert(const std::string& field, const std::string& value, uint32_t id, bool is_string = false);
void erase(const std::string& field);
diff --git a/include/field.h b/include/field.h
index 696c0bb0..b4832043 100644
--- a/include/field.h
+++ b/include/field.h
@@ -864,7 +864,7 @@ struct facet {
struct facet_info_t {
// facet hash => resolved tokens
- std::unordered_map<uint64_t, std::vector<std::string>> hashes;
+ std::unordered_map<uint32_t, std::vector<std::string>> hashes;
bool use_facet_query = false;
bool should_compute_stats = false;
field facet_field{"", "", false};
diff --git a/include/index.h b/include/index.h
index 2bdc6fb6..d045d923 100644
--- a/include/index.h
+++ b/include/index.h
@@ -188,10 +188,6 @@ struct search_args {
};
};
-struct offsets_facet_hashes_t {
- std::unordered_map<std::string, std::vector<uint32_t>> offsets;
-};
-
struct index_record {
size_t position; // position of record in the original request
uint32_t seq_id;
@@ -205,7 +201,8 @@ struct index_record {
bool is_update;
// pre-processed data primed for indexing
- std::unordered_map<std::string, offsets_facet_hashes_t> field_index;
+ std::unordered_map<std::string,
+ std::unordered_map<std::string, std::vector<uint32_t>>> field_index;
int64_t points;
Option<bool> indexed; // indicates if the indexing operation was a success
diff --git a/include/num_tree.h b/include/num_tree.h
index 5406a109..12c51e23 100644
--- a/include/num_tree.h
+++ b/include/num_tree.h
@@ -22,11 +22,23 @@ private:
return ids_t::contains(ids, id);
}
+ struct count_list {
+ count_list(const int64_t& sv, uint32_t facet_count) {
+ facet_value = sv;
+ count = facet_count;
+ }
+
+ int64_t facet_value;
+ uint32_t count;
+ };
+
+ std::vector<count_list> counter_list;
+
public:
~num_tree_t();
- void insert(int64_t value, uint32_t id);
+ void insert(int64_t value, uint32_t id, bool is_facet=false);
void range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len);
@@ -53,4 +65,10 @@ public:
uint32_t* const& context_ids,
size_t& result_ids_len,
uint32_t*& result_ids) const;
+
+ size_t intersect(const uint32_t* result_ids, int result_id_len,
+ int max_facet_count, std::map<int64_t, uint32_t>& found,
+ bool is_wildcard_no_filter_query);
+
+ size_t counter_list_size() const;
};
\ No newline at end of file
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index e00a4c9b..2d49a7df 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -1,8 +1,10 @@
#include "facet_index.h"
#include <tokenizer.h>
#include "string_utils.h"
+#include "array_utils.h"
-uint32_t facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {
+uint32_t facet_index_t::insert(const std::string& field, const std::string& value,
+ uint32_t id, bool is_string) {
auto& facet_index_map = facet_field_map[field].facet_index_map;
uint32_t index;
const auto sv = value.substr(0, 100);
@@ -12,49 +14,56 @@ uint32_t facet_index_t::insert(const std::string& field, const std::string& valu
index = ++count_index;
facet_index_struct fis {};
- fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));
fis.index = index;
+ if(is_string) {
+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));
+ }
facet_index_map.emplace(sv, fis);
} else {
- auto ids = it->id_list_ptr;
- if (!ids_t::contains(ids, id)) {
- ids_t::upsert(ids, id);
- facet_index_map[sv].id_list_ptr = ids;
+ if(is_string) {
+ auto ids = it->id_list_ptr;
+ if (!ids_t::contains(ids, id)) {
+ ids_t::upsert(ids, id);
+ facet_index_map[sv].id_list_ptr = ids;
+ }
}
index = facet_index_map[sv].index;
}
- const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
- //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
- auto& counter_list = facet_field_map[field].counter_list;
- if(counter_list.empty()) {
- counter_list.emplace_back(sv, facet_count);
- } else {
- auto counter_it = counter_list.begin();
- //remove node from list
- for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {
- if(counter_it->facet_value == sv) {
- //found facet in first node
- counter_list.erase(counter_it);
- break;
+ if(is_string) {
+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
+ auto& counter_list = facet_field_map[field].counter_list;
+
+ if(counter_list.empty()) {
+ counter_list.emplace_back(sv, facet_count);
+ } else {
+ auto counter_it = counter_list.begin();
+ //remove node from list
+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {
+ if(counter_it->facet_value == sv) {
+ //found facet in first node
+ counter_list.erase(counter_it);
+ break;
+ }
}
- }
- //find position in list and add node with updated count
- count_list node(sv, facet_count);
+ //find position in list and add node with updated count
+ count_list node(sv, facet_count);
- for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {
- // LOG (INFO) << "inserting in middle or front facet " << node.facet_value
- // << " with count " << node.count;
- if(counter_it->count <= facet_count) {
- counter_list.emplace(counter_it, node);
- break;
+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {
+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value
+ // << " with count " << node.count;
+ if(counter_it->count <= facet_count) {
+ counter_list.emplace(counter_it, node);
+ break;
+ }
+ }
+ if(counter_it == counter_list.end()) {
+ // LOG (INFO) << "inserting at last facet " << node.facet_value
+ // << " with count " << node.count;
+ counter_list.emplace_back(node);
}
- }
- if(counter_it == counter_list.end()) {
- // LOG (INFO) << "inserting at last facet " << node.facet_value
- // << " with count " << node.count;
- counter_list.emplace_back(node);
}
}
@@ -118,9 +127,12 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* result
ids_t::uncompress(ids, id_list);
const auto ids_len = id_list.size();
for(int i = 0; i < result_ids_len; ++i) {
- if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {
- ++count;
- }
+ // if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {
+ // ++count;
+ // }
+ uint32_t* out = nullptr;
+ count = ArrayUtils::and_scalar(id_list.data(), id_list.size(),
+ result_ids, result_ids_len, &out);
}
id_list.clear();
}
diff --git a/src/index.cpp b/src/index.cpp
index 638f7837..26d2fb49 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -656,37 +656,28 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
if(afield.facet) {
if(afield.is_array()) {
- facet_hash_values_t fhashvalues;
-
- if(afield.type == field_types::INT32_ARRAY) {
- for(size_t i = 0; i < document[afield.name].size(); ++i) {
+ facet_hash_values_t fhashvalues;
+ for(size_t i = 0; i < document[afield.name].size(); ++i) {
+ if(afield.type == field_types::INT32_ARRAY) {
int32_t raw_val = document[afield.name][i].get<int32_t>();
value = std::to_string(raw_val);
auto index = facet_index_v4->insert(afield.name, value, seq_id);
fhashvalues.hashes.emplace_back(index);
- }
- } else if(afield.type == field_types::INT64_ARRAY) {
- for(size_t i = 0; i < document[afield.name].size(); ++i) {
+ } else if(afield.type == field_types::INT64_ARRAY) {
int64_t raw_val = document[afield.name][i].get<int64_t>();
value = std::to_string(raw_val);
auto index = facet_index_v4->insert(afield.name, value, seq_id);
fhashvalues.hashes.emplace_back(index);
- }
- } else if(afield.type == field_types::STRING_ARRAY) {
- for(size_t i = 0; i < document[afield.name].size(); ++i) {
+ } else if(afield.type == field_types::STRING_ARRAY) {
value = document[afield.name][i];
- auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id, true);
fhashvalues.hashes.emplace_back(index);
- }
- } else if(afield.type == field_types::FLOAT_ARRAY) {
- for(size_t i = 0; i < document[afield.name].size(); ++i) {
+ } else if(afield.type == field_types::FLOAT_ARRAY) {
float raw_val = document[afield.name][i].get<float>();
value = StringUtils::float_to_str(raw_val);
auto index = facet_index_v4->insert(afield.name, value, seq_id);
fhashvalues.hashes.emplace_back(index);
- }
- } else if(afield.type == field_types::BOOL_ARRAY) {
- for(size_t i = 0; i < document[afield.name].size(); ++i) {
+ } else if(afield.type == field_types::BOOL_ARRAY) {
value = std::to_string(document[afield.name][i].get<bool>());
auto index = facet_index_v4->insert(afield.name, value, seq_id);
fhashvalues.hashes.emplace_back(index);
@@ -694,12 +685,12 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
fhashvalues.length = fhashvalues.hashes.size();
- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- } else {
- facet_dim_index->emplace(seq_id, std::move(fhashvalues));
- }
+ auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
+ } else {
+ facet_dim_index->emplace(seq_id, std::move(fhashvalues));
+ }
} else {
uint32_t fhash;
@@ -715,7 +706,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
else if(afield.type == field_types::STRING) {
value = document[afield.name];
- fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id, true);
}
else if(afield.type == field_types::FLOAT) {
float raw_val = document[afield.name].get<float>();
@@ -740,7 +731,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
max_score = record.points;
}
- for(auto &token_offsets: field_index_it->second.offsets) {
+ for(auto &token_offsets: field_index_it->second) {
token_to_doc_offsets[token_offsets.first].emplace_back(seq_id, record.points, token_offsets.second);
if(afield.infix) {
@@ -776,7 +767,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
(const index_record& record, uint32_t seq_id) {
int32_t value = record.doc[afield.name].get<int32_t>();
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
});
}
@@ -785,7 +776,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
(const index_record& record, uint32_t seq_id) {
int64_t value = record.doc[afield.name].get<int64_t>();
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
});
}
@@ -795,14 +786,14 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
(const index_record& record, uint32_t seq_id) {
float fvalue = record.doc[afield.name].get<float>();
int64_t value = float_to_int64_t(fvalue);
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
});
} else if(afield.type == field_types::BOOL) {
auto num_tree = numerical_index.at(afield.name);
iterate_and_index_numerical_field(iter_batch, afield, [&afield, num_tree]
(const index_record& record, uint32_t seq_id) {
bool value = record.doc[afield.name].get<bool>();
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
});
} else if(afield.type == field_types::GEOPOINT || afield.type == field_types::GEOPOINT_ARRAY) {
auto geo_index = geopoint_index.at(afield.name);
@@ -941,23 +932,23 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
if(afield.type == field_types::INT32_ARRAY) {
const int32_t value = arr_value;
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
}
else if(afield.type == field_types::INT64_ARRAY) {
const int64_t value = arr_value;
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
}
else if(afield.type == field_types::FLOAT_ARRAY) {
const float fvalue = arr_value;
int64_t value = float_to_int64_t(fvalue);
- num_tree->insert(value, seq_id);
+ num_tree->insert(value, seq_id, afield.is_facet());
}
else if(afield.type == field_types::BOOL_ARRAY) {
const bool value = record.doc[afield.name][arr_i];
- num_tree->insert(int64_t(value), seq_id);
+ num_tree->insert(int64_t(value), seq_id, afield.is_facet());
}
}
});
@@ -1164,7 +1155,20 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
size_t mod_value = 100 / facet_sample_percent;
- const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);
+ auto facet_records = 0;
+
+ if(!facet_field.name.empty()) {
+ if(facet_field.is_string()) {
+ facet_records = facet_index_v4->get_facet_count(a_facet.field_name);
+ } else {
+ auto numerical_index_it = numerical_index.find(a_facet.field_name);
+ if(numerical_index_it != numerical_index.end()) {
+ facet_records = numerical_index_it->second->counter_list_size();
+ } else {
+ LOG(ERROR) << "facet " << a_facet.field_name << " not found in numerical index";
+ }
+ }
+ }
if(results_size && facet_records && (facet_records <= 10 || is_wildcard_query) &&
!use_facet_query && group_limit == 0 && no_filters_provided) {
@@ -1172,9 +1176,29 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
a_facet.is_intersected = true;
std::map<std::string, uint32_t> facet_results;
- facet_index_v4->intersect(a_facet.field_name, result_ids,
- results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);
-
+
+ if(facet_field.is_string()) {
+ facet_index_v4->intersect(a_facet.field_name, result_ids,
+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);
+ } else {
+ std::map<int64_t, uint32_t> facet_counts;
+ numerical_index.at(a_facet.field_name)->intersect(result_ids,
+ results_size, max_facet_count, facet_counts, is_wildcard_query & no_filters_provided);
+
+ for(const auto& kv : facet_counts) {
+ std::string val;
+ if(facet_field.is_float()) {
+ val = std::to_string(int64_t_to_float(kv.first));
+ } else if(facet_field.is_bool()) {
+ val = kv.first == 1 ? "true" : "false";
+ } else {
+ val = std::to_string(kv.first);
+ }
+
+ facet_results[val] = kv.second;
+ }
+ }
+
for(const auto& kv : facet_results) {
//range facet processing
if(a_facet.is_range_query) {
diff --git a/src/num_tree.cpp b/src/num_tree.cpp
index 1bcdbc9f..6e433eb1 100644
--- a/src/num_tree.cpp
+++ b/src/num_tree.cpp
@@ -2,7 +2,7 @@
#include "parasort.h"
#include "timsort.hpp"
-void num_tree_t::insert(int64_t value, uint32_t id) {
+void num_tree_t::insert(int64_t value, uint32_t id, bool is_facet) {
if (int64map.count(value) == 0) {
int64map.emplace(value, SET_COMPACT_IDS(compact_id_list_t::create(1, {id})));
} else {
@@ -12,6 +12,43 @@ void num_tree_t::insert(int64_t value, uint32_t id) {
int64map[value] = ids;
}
}
+
+ if(is_facet) {
+
+ const auto facet_count = ids_t::num_ids(int64map.at(value));
+ //LOG(INFO) << "Facet count in facet " << value << " : " << facet_count;
+
+ if(counter_list.empty()) {
+ counter_list.emplace_back(value, facet_count);
+ } else {
+ auto counter_it = counter_list.begin();
+ //remove node from list
+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {
+ if(counter_it->facet_value == value) {
+ //found facet in first node
+ counter_list.erase(counter_it);
+ break;
+ }
+ }
+
+ //find position in list and add node with updated count
+ count_list node(value, facet_count);
+
+ for(counter_it = counter_list.begin(); counter_it != counter_list.end(); ++counter_it) {
+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value
+ // << " with count " << node.count;
+ if(counter_it->count <= facet_count) {
+ counter_list.emplace(counter_it, node);
+ break;
+ }
+ }
+ if(counter_it == counter_list.end()) {
+ // LOG (INFO) << "inserting at last facet " << node.facet_value
+ // << " with count " << node.count;
+ counter_list.emplace_back(node);
+ }
+ }
+ }
}
void num_tree_t::range_inclusive_search(int64_t start, int64_t end, uint32_t** ids, size_t& ids_len) {
@@ -322,8 +359,53 @@ size_t num_tree_t::size() {
return int64map.size();
}
+size_t num_tree_t::counter_list_size() const {
+ return counter_list.size();
+}
+
num_tree_t::~num_tree_t() {
for(auto& kv: int64map) {
ids_t::destroy_list(kv.second);
}
}
+
+size_t num_tree_t::intersect(const uint32_t* result_ids, int result_ids_len, int max_facet_count,
+ std::map<int64_t, uint32_t>& found, bool is_wildcard_no_filter_query) {
+ //LOG (INFO) << "intersecting field " << field;
+
+ // LOG (INFO) << "int64map size " << int64map.size()
+ // << " , counter_list size " << counter_list.size();
+
+ std::vector<uint32_t> id_list;
+ for(const auto& counter_list_it : counter_list) {
+ // LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value
+ // << " having total count " << counter_list_it.count;
+ uint32_t count = 0;
+
+ if(is_wildcard_no_filter_query) {
+ count = counter_list_it.count;
+ } else {
+ auto ids = int64map.at(counter_list_it.facet_value);
+ ids_t::uncompress(ids, id_list);
+ const auto ids_len = id_list.size();
+ for(int i = 0; i < result_ids_len; ++i) {
+ // if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {
+ // ++count;
+ // }
+ uint32_t* out = nullptr;
+ count = ArrayUtils::and_scalar(id_list.data(), id_list.size(),
+ result_ids, result_ids_len, &out);
+ }
+ id_list.clear();
+ }
+
+ if(count) {
+ found[counter_list_it.facet_value] = count;
+ if(found.size() == max_facet_count) {
+ break;
+ }
+ }
+ }
+
+ return found.size();
+}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 8872759f..649a91d0 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -252,7 +252,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_FLOAT_EQ(5, results["facet_counts"][0]["stats"]["total_values"].get<size_t>());
// check for "0" case
- ASSERT_STREQ("0", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("0.000000", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
ASSERT_EQ(1, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
// facet query on a float field
@@ -506,8 +506,8 @@ TEST_F(CollectionFacetingTest, FacetCountsFloatPrecision) {
ASSERT_STREQ("points", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("113.4", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("113.4",results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("113.400002", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("113.400002",results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}