Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

hybrid approach preserving both structures

krunal1313595a9b493353
diff --git a/include/facet_index.h b/include/facet_index.h
index fecea8e7..29243f8d 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -2,7 +2,6 @@
#include "ids_t.h"
#include "tsl/htrie_map.h"
-#include <list>
class facet_index_t {
private:
@@ -16,13 +15,18 @@ private:
uint32_t count;
};
+ struct facet_index_struct {
+ void* id_list_ptr;
+ uint32_t index;
+ };
+
struct facet_index_counter {
- tsl::htrie_map<char, void*> facet_index_map;
- std::list<count_list> counter_list;
-
+ tsl::htrie_map<char, facet_index_struct> facet_index_map;
+ std::vector<count_list> counter_list;
+
~facet_index_counter() {
for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {
- ids_t::destroy_list(it.value());
+ ids_t::destroy_list(it.value().id_list_ptr);
}
facet_index_map.clear();
@@ -32,23 +36,23 @@ private:
};
std::map<std::string, facet_index_counter> facet_field_map;
+ uint32_t count_index = 0;
public:
facet_index_t() = default;
~facet_index_t();
- void insert(const std::string& field, const std::string& value, uint32_t id);
+ uint32_t insert(const std::string& field, const std::string& value, uint32_t id);
void erase(const std::string& field);
bool contains(const std::string& field);
- size_t size();
+ size_t get_facet_count(const std::string& field);
int intersect(const std::string& val, const uint32_t* result_ids, int result_id_len,
int max_facet_count, std::map<std::string, uint32_t>& found);
- int get_facet(const std::string& field, const std::vector<std::string>& searched_tokens,
- std::vector<std::string>& facets);
+ std::string get_facet_by_count_index(const std::string& field, uint32_t count_index);
};
\ No newline at end of file
diff --git a/include/field.h b/include/field.h
index 1fa358ae..b42f6e0f 100644
--- a/include/field.h
+++ b/include/field.h
@@ -637,11 +637,10 @@ struct facet {
const std::string field_name;
spp::sparse_hash_map<std::string, facet_count_t> result_map;
// used for facet value query
- //spp::sparse_hash_map<uint64_t, std::vector<std::string>> hash_tokens;
- spp::sparse_hash_map<std::string, std::vector<std::string>> facet_tokens;
+ spp::sparse_hash_map<uint32_t, std::vector<std::string>> hash_tokens;
// used for faceting grouped results
- //spp::sparse_hash_map<uint64_t, spp::sparse_hash_set<uint64_t>> hash_groups;
+ spp::sparse_hash_map<uint32_t, spp::sparse_hash_set<uint32_t>> hash_groups;
facet_stats_t stats;
@@ -652,6 +651,8 @@ struct facet {
bool sampled = false;
+ bool is_intersected = false;
+
bool get_range(std::string key, std::pair<int64_t, std::string>& range_pair)
{
if(facet_range_map.empty())
@@ -680,9 +681,8 @@ struct facet {
struct facet_info_t {
// facet hash => resolved tokens
- //std::unordered_map<uint64_t, std::vector<std::string>> hashes;
+ std::unordered_map<uint64_t, std::vector<std::string>> hashes;
//facet name => resolved tokens
- std::unordered_map<std::string, std::vector<std::string>> facet_tokens;
bool use_facet_query = false;
bool should_compute_stats = false;
field facet_field{"", "", false};
@@ -701,11 +701,12 @@ struct facet_value_t {
struct facet_hash_values_t {
uint32_t length = 0;
- uint64_t* hashes = nullptr;
+ //uint32_t* hashes = nullptr;
+ std::vector<uint32_t> hashes;
facet_hash_values_t() {
length = 0;
- hashes = nullptr;
+ //hashes = nullptr;
}
facet_hash_values_t(facet_hash_values_t&& hash_values) noexcept {
@@ -713,17 +714,20 @@ struct facet_hash_values_t {
hashes = hash_values.hashes;
hash_values.length = 0;
- hash_values.hashes = nullptr;
+ //hash_values.hashes = nullptr;
+ hash_values.hashes.clear();
}
facet_hash_values_t& operator=(facet_hash_values_t&& other) noexcept {
if (this != &other) {
- delete[] hashes;
+ //delete[] hashes;
+ hashes.clear();
hashes = other.hashes;
length = other.length;
- other.hashes = nullptr;
+ //other.hashes = nullptr;
+ other.hashes.clear();
other.length = 0;
}
@@ -731,8 +735,9 @@ struct facet_hash_values_t {
}
~facet_hash_values_t() {
- delete [] hashes;
- hashes = nullptr;
+ //delete [] hashes;
+ //hashes = nullptr;
+ hashes.clear();
}
uint64_t size() const {
@@ -740,6 +745,7 @@ struct facet_hash_values_t {
}
uint64_t back() const {
- return hashes[length - 1];
+ //return hashes[length - 1];
+ return hashes.back();
}
};
diff --git a/include/index.h b/include/index.h
index cf8f76c4..953dcf6d 100644
--- a/include/index.h
+++ b/include/index.h
@@ -33,10 +33,10 @@
#include "facet_index.h"
static constexpr size_t ARRAY_FACET_DIM = 4;
-//using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;
-//using single_val_facet_map_t = spp::sparse_hash_map<uint32_t, uint64_t>;
-//using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;
-//using array_mapped_single_val_facet_t = std::array<single_val_facet_map_t*, ARRAY_FACET_DIM>;
+using facet_map_t = spp::sparse_hash_map<uint32_t, facet_hash_values_t>;
+using single_val_facet_map_t = spp::sparse_hash_map<uint32_t, uint32_t>;
+using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;
+using array_mapped_single_val_facet_t = std::array<single_val_facet_map_t*, ARRAY_FACET_DIM>;
static constexpr size_t ARRAY_INFIX_DIM = 4;
using array_mapped_infix_t = std::vector<tsl::htrie_set<char>*>;
@@ -190,7 +190,7 @@ struct search_args {
struct offsets_facet_hashes_t {
std::unordered_map<std::string, std::vector<uint32_t>> offsets;
- std::vector<uint64_t> facet_hashes;
+ //std::vector<uint32_t> facet_hashes;
};
struct index_record {
@@ -309,11 +309,12 @@ private:
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t*>*> geo_array_index;
// facet_field => (seq_id => values)
- //spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;
+ spp::sparse_hash_map<std::string, array_mapped_facet_t> facet_index_v3;
+
facet_index_t* facet_index_v4 = nullptr;
// facet_field => (seq_id => hash)
- //spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;
+ spp::sparse_hash_map<std::string, array_mapped_single_val_facet_t> single_val_facet_index_v3;
// sort_field => (seq_id => value)
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;
@@ -491,15 +492,15 @@ private:
static void tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes);
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
+ std::vector<uint32_t>& facet_hashes*/);
static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes);
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
+ std::vector<uint32_t>& facet_hashes*/);
void collate_included_ids(const std::vector<token_t>& q_included_tokens,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
@@ -507,7 +508,7 @@ private:
static uint64_t facet_token_hash(const field & a_field, const std::string &token);
- static void compute_facet_stats(facet &a_facet, std::string raw_value, const std::string & field_type);
+ static void compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type);
static void handle_doc_ops(const tsl::htrie_map<char, field>& search_schema,
nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc);
diff --git a/src/collection.cpp b/src/collection.cpp
index eb4ae625..b4e5849d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1920,22 +1920,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
facet_result["counts"] = nlohmann::json::array();
std::vector<facet_value_t> facet_values;
- // std::vector<std::pair<uint64_t, facet_count_t>> facet_hash_counts;
-
- // for (const auto & kv : a_facet.result_map) {
- // facet_hash_counts.emplace_back(kv);
- // }
- auto the_field = search_schema.at(a_facet.field_name);
- // keep only top K facets
- //auto max_facets = std::min(max_facet_values, facet_hash_counts.size());
- auto max_facets = std::min(max_facet_values, a_facet.result_map.size());
- // std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
- // facet_hash_counts.end(), Collection::facet_count_compare);
- //LOG (INFO) << "found facet size " << a_facet.result_map.size();
- for(auto& kv : a_facet.result_map) {
-
- if(a_facet.is_range_query){
+ if(a_facet.is_range_query){
+ for(auto kv : a_facet.result_map){
+
auto facet_range_iter = a_facet.facet_range_map.find(kv.first);
if(facet_range_iter != a_facet.facet_range_map.end()){
auto & facet_count = kv.second;
@@ -1945,39 +1933,78 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
else{
LOG (ERROR) << "range_id not found in result map.";
}
- } else {
- //facet_value_t facet_value = { kv.first, std::string(), kv.second.count};
- //facet_values.emplace_back(facet_value);
+ }
+ }
+
+ if(a_facet.is_intersected) {
+ std::vector<std::pair<std::string, facet_count_t>> facet_counts;
+
+ for (const auto & kv : a_facet.result_map) {
+ facet_counts.emplace_back(std::make_pair(kv.first, kv.second));
+ }
+
+ auto max_facets = std::min(max_facet_values, facet_counts.size());
+ std::sort(facet_counts.begin(), facet_counts.end(),
+ [&](const auto& p1, const auto& p2) {
+ return p1.second.count > p2.second.count;
+ });
+
+ for(int i = 0; i < max_facets; ++i) {
+ if(a_facet.is_range_query) {
+ break;
+ }
+ const auto& kv = facet_counts[i];
+ facet_value_t facet_value = { kv.first, kv.first, kv.second.count};
+ facet_values.emplace_back(facet_value);
+ }
+ } else {
+
+ std::vector<std::pair<uint32_t, facet_count_t>> facet_hash_counts;
+
+ for (const auto & kv : a_facet.result_map) {
+ facet_hash_counts.emplace_back(std::make_pair(std::stoul(kv.first), kv.second));
+ }
+
+ auto the_field = search_schema.at(a_facet.field_name);
+ // keep only top K facets
+ auto max_facets = std::min(max_facet_values, facet_hash_counts.size());
+
+ std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
+ facet_hash_counts.end(), Collection::facet_count_compare);
+ for(size_t fi = 0; fi < max_facets; fi++) {
+
+ if(a_facet.is_range_query){
+ break;
+ }
// remap facet value hash with actual string
- // auto & kv = facet_hash_counts[fi];
- // auto & facet_count = kv.second;
- // // fetch actual facet value from representative doc id
- // const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);
- // nlohmann::json document;
- // const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
- // if(!document_op.ok()) {
- // LOG(ERROR) << "Facet fetch error. " << document_op.error();
- // continue;
- // }
- //std::string value;
- // bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);
- // if(!facet_found) {
- // continue;
- // }
- std::string value = kv.first;
+ auto & kv = facet_hash_counts[fi];
+ auto & facet_count = kv.second;
+ // fetch actual facet value from representative doc id
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);
+ nlohmann::json document;
+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
+ if(!document_op.ok()) {
+ LOG(ERROR) << "Facet fetch error. " << document_op.error();
+ continue;
+ }
+ std::string value;
+ bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);
+ if(!facet_found) {
+ continue;
+ }
std::unordered_map<std::string, size_t> ftoken_pos;
- //std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];
- std::vector<string>& ftokens = a_facet.facet_tokens[kv.first];
+ std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];
+ //LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();
for(size_t ti = 0; ti < ftokens.size(); ti++) {
- // if(the_field.is_bool()) {
- // if(ftokens[ti] == "1") {
- // ftokens[ti] = "true";
- // } else {
- // ftokens[ti] = "false";
- // }
- // }
+ if(the_field.is_bool()) {
+ if(ftokens[ti] == "1") {
+ ftokens[ti] = "true";
+ } else {
+ ftokens[ti] = "false";
+ }
+ }
const std::string& resolved_token = ftokens[ti];
ftoken_pos[resolved_token] = ti;
}
@@ -2006,6 +2033,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
}
}
+ //LOG(INFO) << "token_offsets size " << token_offsets.size();
auto offset_it = token_offsets.begin();
size_t i = 0;
std::stringstream highlightedss;
@@ -2035,7 +2063,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
highlightedss << value[i];
i++;
}
- facet_value_t facet_value = {value, highlightedss.str(), kv.second.count};
+ facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};
facet_values.emplace_back(facet_value);
}
}
@@ -4617,7 +4645,7 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector
return Option<bool>(400, error);
}
- std::vector<std::tuple<int64_t, int64_t, std::string>> tupVec;
+ std::vector<std::tuple<std::string, std::string, std::string>> tupVec;
auto& range_map = a_facet.facet_range_map;
for(const auto& range : result){
@@ -4626,7 +4654,6 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector
std::string error = "Facet range value is not valid.";
return Option<bool>(400, error);
}
-
auto pos1 = range.find(":");
std::string range_val = range.substr(0, pos1);
@@ -4654,8 +4681,8 @@ Option<bool> Collection::parse_facet(const std::string& facet_field, std::vector
for(const auto& tup : tupVec){
- const std::string& lower_range = std::to_string(std::get<0>(tup));
- const std::string& upper_range = std::to_string(std::get<1>(tup));
+ const auto& lower_range = std::get<0>(tup);
+ const auto& upper_range = std::get<1>(tup);
const std::string& range_val = std::get<2>(tup);
//check if ranges are continous or not
if((!range_map.empty()) && (range_map.find(lower_range)== range_map.end())){
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index 4a0e147f..0421653a 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -1,23 +1,30 @@
#include "facet_index.h"
+#include <tokenizer.h>
+#include "string_utils.h"
-void facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {
-
+uint32_t facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {
auto& facet_index_map = facet_field_map[field].facet_index_map;
-
+ uint32_t index;
const auto sv = value.substr(0, 100);
if(facet_index_map.count(sv) == 0) {
- facet_index_map.emplace(sv, SET_COMPACT_IDS(compact_id_list_t::create(1, {id})));
+ index = ++count_index;
+
+ facet_index_struct fis {};
+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));
+ fis.index = index;
+ facet_index_map.emplace(sv, fis);
}else {
- auto ids = facet_index_map[sv];
+ auto ids = facet_index_map[sv].id_list_ptr;
if (!ids_t::contains(ids, id)) {
ids_t::upsert(ids, id);
- facet_index_map[sv] = ids;
+ facet_index_map[sv].id_list_ptr = ids;
}
+ index = facet_index_map[sv].index;
}
-
- const auto facet_count = ids_t::num_ids(facet_index_map.at(sv));
+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
//LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
auto& counter_list = facet_field_map[field].counter_list;
+ count_list node(sv, facet_count);
if(counter_list.empty()) {
counter_list.emplace_back(count_list(sv, facet_count));
@@ -49,6 +56,8 @@ void facet_index_t::insert(const std::string& field, const std::string& value, u
counter_list.emplace_back(node);
}
}
+
+ return index;
}
bool facet_index_t::contains(const std::string& field) {
@@ -68,8 +77,14 @@ void facet_index_t::erase(const std::string& field) {
}
}
-size_t facet_index_t::size() {
- return facet_field_map.size();
+size_t facet_index_t::get_facet_count(const std::string& field) {
+ const auto it = facet_field_map.find(field);
+
+ if(it == facet_field_map.end()) {
+ return 0;
+ }
+
+ return it->second.counter_list.size();
}
//returns the count of matching seq_ids from result array
@@ -90,12 +105,11 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_id
// << " , counter_list size " << counter_list.size();
std::vector<uint32_t> id_list;
- const auto max_facets = std::min((int)counter_list.size(), max_facet_count);
for(const auto& counter_list_it : counter_list) {
- //LOG (INFO) << "checking ids in facet_value " << counter_list_it->facet_value
- // << " having total count " << counter_list_it->count;
+ // LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value
+ // << " having total count " << counter_list_it.count;
- auto ids = facet_index_map.at(counter_list_it.facet_value);
+ auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;
ids_t::uncompress(ids, id_list);
const auto ids_len = id_list.size();
int count = 0;
@@ -108,8 +122,7 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_id
if(count) {
found[counter_list_it.facet_value] = count;
-
- if(found.size() == max_facets) {
+ if(found.size() == 2*max_facet_count) {
break;
}
}
@@ -120,34 +133,22 @@ int facet_index_t::intersect(const std::string& field, const uint32_t* result_id
return found.size();
}
-int facet_index_t::get_facet(const std::string& field, const std::vector<std::string>& searched_tokens,
- std::vector<std::string>& facets) {
+std::string facet_index_t::get_facet_by_count_index(const std::string& field, uint32_t count_index) {
const auto& facet_field_it = facet_field_map.find(field);
if(facet_field_it == facet_field_map.end()) {
- return 0;
+ return "";
}
-
+ std::string result = "";
auto facet_index_map = facet_field_it->second.facet_index_map;
- for(const auto& token : searched_tokens) {
- auto token_string = token;
- std::transform(token_string.begin(), token_string.end(), token_string.begin(), ::tolower);
-
- for(auto facet_index_map_it = facet_index_map.begin();
- facet_index_map_it != facet_index_map.end(); ++facet_index_map_it) {
-
- auto facet_string = facet_index_map_it.key();
- std::transform(facet_string.begin(), facet_string.end(), facet_string.begin(), ::tolower);
-
- if(facet_string.find(token_string) != std::string::npos) {
- facets.emplace_back(facet_index_map_it.key());
- }
+ for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {
+ if(it.value().index == count_index) {
+ result = it.key();
}
}
-
- return facets.size();
+ return result;
}
facet_index_t::~facet_index_t() {
diff --git a/src/index.cpp b/src/index.cpp
index 67943ff7..1f12a6ad 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -103,7 +103,7 @@ Index::Index(const std::string& name, const uint32_t collection_id, const Store*
}
if(a_field.facet) {
- //initialize_facet_indexes(a_field);
+ initialize_facet_indexes(a_field);
}
// initialize for non-string facet fields
@@ -186,23 +186,23 @@ Index::~Index() {
str_sort_index.clear();
- //for(auto& field_name_facet_map_array: facet_index_v3) {
- // for(auto& facet_map: field_name_facet_map_array.second) {
- // delete facet_map;
- // facet_map = nullptr;
- // }
- //}//
+ for(auto& field_name_facet_map_array: facet_index_v3) {
+ for(auto& facet_map: field_name_facet_map_array.second) {
+ delete facet_map;
+ facet_map = nullptr;
+ }
+ }
- //facet_index_v3.clear();
+ facet_index_v3.clear();
- ////for(auto& field_name_facet_map_array: single_val_facet_index_v3) {
- // for(auto& facet_map: field_name_facet_map_array.second) {
- //// delete facet_map;
- // facet_map = nullptr;
- // }
- //}
+ for(auto& field_name_facet_map_array: single_val_facet_index_v3) {
+ for(auto& facet_map: field_name_facet_map_array.second) {
+ delete facet_map;
+ facet_map = nullptr;
+ }
+ }
- //single_val_facet_index_v3.clear();
+ single_val_facet_index_v3.clear();
delete facet_index_v4;
delete seq_ids;
@@ -296,7 +296,7 @@ void Index::compute_token_offsets_facets(index_record& record,
tokenize_string_array_with_facets(strings, is_facet, the_field,
local_symbols_to_index, local_token_separators,
- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);
} else {
std::string text;
@@ -312,7 +312,7 @@ void Index::compute_token_offsets_facets(index_record& record,
tokenize_string_with_facets(text, is_facet, the_field,
local_symbols_to_index, local_token_separators,
- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);
}
}
@@ -321,15 +321,15 @@ void Index::compute_token_offsets_facets(index_record& record,
if(the_field.type == field_types::STRING) {
tokenize_string_with_facets(document[field_name], is_facet, the_field,
local_symbols_to_index, local_token_separators,
- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);
} else {
tokenize_string_array_with_facets(document[field_name], is_facet, the_field,
local_symbols_to_index, local_token_separators,
- offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);
+ offset_facet_hashes.offsets/*, offset_facet_hashes.facet_hashes*/);
}
}
- if(!offset_facet_hashes.offsets.empty() || !offset_facet_hashes.facet_hashes.empty()) {
+ if(!offset_facet_hashes.offsets.empty() /*|| !offset_facet_hashes.facet_hashes.empty()*/) {
record.field_index.emplace(field_name, std::move(offset_facet_hashes));
}
}
@@ -652,98 +652,102 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
if(field_index_it == record.field_index.end()) {
continue;
}
+
+ std::string value;
if(afield.facet) {
- //if(afield.is_array()) {
- // facet_hash_values_t fhashvalues;
- // fhashvalues.length = field_index_it->second.facet_hashes.size();
- // fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];
-
- // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {
- // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];
- // }
-
- // auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- // if(facet_dim_index == nullptr) {
- // LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- // } else {
- // facet_dim_index->emplace(seq_id, std::move(fhashvalues));
- // }
- //} else {
- // uint64_t fhash;
- // fhash = field_index_it->second.facet_hashes[0];
-
- // auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- // if(facet_dim_index == nullptr) {
- // LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- // } else {
- // facet_dim_index->emplace(seq_id, std::move(fhash));
- // }
- //}
- std::string value;
- if(afield.type == field_types::INT32) {
- int32_t raw_val = document[afield.name].get<int32_t>();
- value = std::to_string(raw_val);
- facet_index_v4->insert(afield.name, value, seq_id);
- }
- else if(afield.type == field_types::INT32_ARRAY) {
- for(int i = 0; i < document[afield.name].size(); ++i) {
- int32_t raw_val = document[afield.name][i].get<int32_t>();
- value = std::to_string(raw_val);
- facet_index_v4->insert(afield.name, value, seq_id);
+ if(afield.is_array()) {
+ facet_hash_values_t fhashvalues;
+
+ if(afield.type == field_types::INT32_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ int32_t raw_val = document[afield.name][i].get<int32_t>();
+ value = std::to_string(raw_val);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
}
- }
- else if(afield.type == field_types::INT64) {
- int64_t raw_val = document[afield.name].get<int64_t>();
- value = std::to_string(raw_val);
- facet_index_v4->insert(afield.name, value, seq_id);
- }
- else if(afield.type == field_types::INT64_ARRAY) {
- for(int i = 0; i < document[afield.name].size(); ++i) {
- int64_t raw_val = document[afield.name][i].get<int64_t>();
+ else if(afield.type == field_types::INT64_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ int64_t raw_val = document[afield.name][i].get<int64_t>();
+ value = std::to_string(raw_val);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ else if(afield.type == field_types::STRING_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ value = document[afield.name][i];
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ else if(afield.type == field_types::FLOAT_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ float raw_val = document[afield.name][i].get<float>();
+ value = StringUtils::float_to_str(raw_val);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ else if(afield.type == field_types::BOOL_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ value = std::to_string(document[afield.name][i].get<bool>());
+ //value = (value == "1") ? "true" : "false";
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ fhashvalues.length = fhashvalues.hashes.size();
+ //LOG(INFO) << "fhashvalues.length " << fhashvalues.length;
+ //fhashvalues.length = field_index_it->second.facet_hashes.size();
+ //fhashvalues.hashes = new uint32_t[field_index_it->second.facet_hashes.size()];
+
+ // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {
+ // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];
+ // }
+
+ auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
+ } else {
+ facet_dim_index->emplace(seq_id, std::move(fhashvalues));
+ }
+ } else {
+ uint32_t fhash;
+
+ if(afield.type == field_types::INT32) {
+ int32_t raw_val = document[afield.name].get<int32_t>();
value = std::to_string(raw_val);
- facet_index_v4->insert(afield.name, value, seq_id);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
}
- }
- else if(afield.type == field_types::STRING) {
- value = document[afield.name];
- facet_index_v4->insert(afield.name, value, seq_id);
- }
- else if(afield.type == field_types::STRING_ARRAY) {
- for(int i = 0; i < document[afield.name].size(); ++i) {
- value = document[afield.name][i];
- facet_index_v4->insert(afield.name, value, seq_id);
+ else if(afield.type == field_types::INT64) {
+ int64_t raw_val = document[afield.name].get<int64_t>();
+ value = std::to_string(raw_val);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
}
- }
- else if(afield.type == field_types::FLOAT) {
- float raw_val = document[afield.name].get<float>();
- value = StringUtils::float_to_str(raw_val);
- if(value != "0") {
- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
+ else if(afield.type == field_types::STRING) {
+ value = document[afield.name];
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
}
- facet_index_v4->insert(afield.name, value, seq_id);
- }
- else if(afield.type == field_types::FLOAT) {
- for(int i = 0; i < document[afield.name].size(); ++i) {
- float raw_val = document[afield.name][i].get<float>();
+ else if(afield.type == field_types::FLOAT) {
+ float raw_val = document[afield.name].get<float>();
value = StringUtils::float_to_str(raw_val);
- if(value != "0") {
- value.erase ( value.find_last_not_of('0') + 1, std::string::npos ); // remove trailing zeros
- }
- facet_index_v4->insert(afield.name, value, seq_id);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
}
- }
- else if(afield.type == field_types::BOOL) {
- value = std::to_string(document[afield.name].get<bool>());
- value = (value == "1") ? "true" : "false";
- facet_index_v4->insert(afield.name, value, seq_id);
- }
- else if(afield.type == field_types::BOOL) {
- for(int i = 0; i < document[afield.name].size(); ++i) {
- value = std::to_string(document[afield.name][i].get<bool>());
- value = (value == "1") ? "true" : "false";
- facet_index_v4->insert(afield.name, value, seq_id);
+ else if(afield.type == field_types::BOOL) {
+ value = std::to_string(document[afield.name].get<bool>());
+ //value = (value == "1") ? "true" : "false";
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
}
+ //fhash = field_index_it->second.facet_hashes[0];
+
+ auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
+ } else {
+ facet_dim_index->emplace(seq_id, std::move(fhash));
+ }
}
}
@@ -1055,8 +1059,8 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &token
void Index::tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes) {
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
+ std::vector<uint32_t>& facet_hashes*/) {
Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
std::string token;
@@ -1075,15 +1079,14 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,
token_to_offsets[token].push_back(token_index + 1);
last_token = token;
-
- if(is_facet) {
- uint64_t token_hash = Index::facet_token_hash(a_field, token);
- if(token_index == 0) {
- facet_hash = token_hash;
- } else {
- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
- }
- }
+ // if(is_facet) {
+ // uint64_t token_hash = Index::facet_token_hash(a_field, token);
+ // if(token_index == 0) {
+ // facet_hash = token_hash;
+ // } else {
+ // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
+ // }
+ // }
}
if(!token_to_offsets.empty()) {
@@ -1091,17 +1094,17 @@ void Index::tokenize_string_with_facets(const std::string& text, bool is_facet,
token_to_offsets[last_token].push_back(0);
}
- if(is_facet) {
- facet_hashes.push_back(facet_hash);
- }
+ // if(is_facet) {
+ // facet_hashes.push_back(facet_hash);
+ // }
}
void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes) {
+ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
+ std::vector<uint32_t>& facet_hashes*/) {
for(size_t array_index = 0; array_index < strings.size(); array_index++) {
const std::string& str = strings[array_index];
@@ -1126,19 +1129,19 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st
token_set.insert(token);
last_token = token;
- if(is_facet) {
- uint64_t token_hash = Index::facet_token_hash(a_field, token);
- if(token_index == 0) {
- facet_hash = token_hash;
- } else {
- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
- }
- }
+ // if(is_facet) {
+ // uint64_t token_hash = Index::facet_token_hash(a_field, token);
+ // if(token_index == 0) {
+ // facet_hash = token_hash;
+ // } else {
+ // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
+ // }
+ // }
}
- if(is_facet) {
- facet_hashes.push_back(facet_hash);
- }
+ // if(is_facet) {
+ // facet_hashes.push_back(facet_hash);
+ // }
if(token_set.empty()) {
continue;
@@ -1157,22 +1160,22 @@ void Index::tokenize_string_array_with_facets(const std::vector<std::string>& st
}
}
void Index::initialize_facet_indexes(const field& facet_field) {
- // if(facet_field.is_array()) {
- // array_mapped_facet_t facet_array;
- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- // facet_array[i] = new facet_map_t();
- // }
- // facet_index_v3.emplace(facet_field.name, facet_array);
- // } else {
- // array_mapped_single_val_facet_t facet_array;
- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- // facet_array[i] = new single_val_facet_map_t();
- // }
- // single_val_facet_index_v3.emplace(facet_field.name, facet_array);
- // }
+ if(facet_field.is_array()) {
+ array_mapped_facet_t facet_array;
+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
+ facet_array[i] = new facet_map_t();
+ }
+ facet_index_v3.emplace(facet_field.name, facet_array);
+ } else {
+ array_mapped_single_val_facet_t facet_array;
+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
+ facet_array[i] = new single_val_facet_map_t();
+ }
+ single_val_facet_index_v3.emplace(facet_field.name, facet_array);
+ }
}
-void Index::compute_facet_stats(facet &a_facet, std::string raw_value, const std::string & field_type) {
+void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {
if(field_type == field_types::INT32 || field_type == field_types::INT32_ARRAY) {
int32_t val = std::stoi(raw_value);
if (val < a_facet.stats.fvmin) {
@@ -1184,7 +1187,7 @@ void Index::compute_facet_stats(facet &a_facet, std::string raw_value, const std
a_facet.stats.fvsum += val;
a_facet.stats.fvcount++;
} else if(field_type == field_types::INT64 || field_type == field_types::INT64_ARRAY) {
- int64_t val = std::stol(raw_value);
+ int64_t val = std::stoll(raw_value);
if(val < a_facet.stats.fvmin) {
a_facet.stats.fvmin = val;
}
@@ -1217,126 +1220,136 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
auto& a_facet = facets[findex];
const auto& facet_field = facet_infos[findex].facet_field;
const bool use_facet_query = facet_infos[findex].use_facet_query;
- //const auto& fquery_hashes = facet_infos[findex].hashes;
- const auto& fquery_facet_tokens = facet_infos[findex].facet_tokens;
+ const auto& fquery_hashes = facet_infos[findex].hashes;
const bool should_compute_stats = facet_infos[findex].should_compute_stats;
auto sort_index_it = sort_index.find(a_facet.field_name);
size_t mod_value = 100 / facet_sample_percent;
- // facet_map_t::iterator facet_map_it;
- // single_val_facet_map_t::iterator single_facet_map_it;
- // uint64_t fhash = 0;
- // size_t facet_hash_count = 1;
- // const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- // const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
- std::map<std::string, uint32_t> facet_results;
- facet_index_v4->intersect(a_facet.field_name, result_ids,
- results_size, max_facet_count, facet_results);
- //LOG(INFO) << "facet_results size " << facet_results.size();
-
- for(const auto& kv : facet_results) {
- //range facet processing
- if(a_facet.is_range_query) {
- const auto doc_val = kv.first;
- std::pair<std::string, std::string> range_pair {};
- if(a_facet.get_range(doc_val, range_pair)) {
- const auto& range_id = range_pair.first;
- facet_count_t& facet_count = a_facet.result_map[range_id];
+ const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);
+ const auto records_to_search = facet_records * results_size;
+
+ //LOG(INFO) << "results size " << results_size;
+ //LOG(INFO) << "records_to_search : " << records_to_search;
+ if(use_facet_query == false && group_limit == 0 && records_to_search != 0
+ && records_to_search < 10000) {
+ //LOG(INFO) << "Using intersection to find facets";
+ a_facet.is_intersected = true;
+
+ std::map<std::string, uint32_t> facet_results;
+ facet_index_v4->intersect(a_facet.field_name, result_ids,
+ results_size, max_facet_count, facet_results);
+ //LOG(INFO) << "facet_results size " << facet_results.size();
+
+ for(const auto& kv : facet_results) {
+ //range facet processing
+ if(a_facet.is_range_query) {
+ const auto doc_val = kv.first;
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count = kv.second;
+ }
+ } else {
+ facet_count_t& facet_count = a_facet.result_map[kv.first];
facet_count.count = kv.second;
}
- } else if(use_facet_query) {
- if (fquery_facet_tokens.find(kv.first) != fquery_facet_tokens.end()) {
- a_facet.facet_tokens[kv.first] = fquery_facet_tokens.at(kv.first);
-
- facet_count_t& facet_count = a_facet.result_map[kv.first];
- facet_count.count = kv.second;
+
+ if(should_compute_stats) {
+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;
+ for(int i = 0; i < kv.second; ++i) {
+ compute_facet_stats(a_facet, kv.first, facet_field.type);
}
- } else {
- facet_count_t& facet_count = a_facet.result_map[kv.first];
- facet_count.count = kv.second;
- }
+ }
+ }
+ } else {
+ facet_map_t::iterator facet_map_it;
+ single_val_facet_map_t::iterator single_facet_map_it;
+ uint32_t fhash = 0;
+ size_t facet_hash_count = 1;
+
+ const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
- if(should_compute_stats) {
- //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;
- for(int i = 0; i < kv.second; ++i) {
- compute_facet_stats(a_facet, kv.first, facet_field.type);
+ for(size_t i = 0; i < results_size; i++) {
+ // if sampling is enabled, we will skip a portion of the results to speed up things
+ if(estimate_facets) {
+ if(i % mod_value != 0) {
+ continue;
+ }
}
- }
- }
- }
- // for(size_t i = 0; i < results_size; i++) {
- // // if sampling is enabled, we will skip a portion of the results to speed up things
- // if(estimate_facets) {
- // if(i % mod_value != 0) {
- // continue;
- // }
- // }
-
- // uint32_t doc_seq_id = result_ids[i];
- // if(facet_field.is_array()) {
- // const auto& field_facet_mapping = field_facet_mapping_it->second;
- // facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- // if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- // continue;
- // }
- // facet_hash_count = facet_map_it->second.size();
- // } else {
- // const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- // single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- // if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- // continue;
- // }
- // facet_hash_count = 1;
- // fhash = single_facet_map_it->second;
- // }
+ uint32_t doc_seq_id = result_ids[i];
+ if(facet_field.is_array()) {
+ const auto& field_facet_mapping = field_facet_mapping_it->second;
+ facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
- // const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
- // //for(size_t j = 0; j < facet_hash_count; j++) {
- // // if(facet_field.is_array()) {
- // // fhash = facet_map_it->second.hashes[j];
- // // }
- // const auto& doc_id = found_doc_seq_ids[j].second;
- // if(should_compute_stats) {
- // compute_facet_stats(a_facet, doc_id, facet_field.type);
- // }
- // if(a_facet.is_range_query) {
- // auto sort_index_it = sort_index.find(a_facet.field_name);
- // if(sort_index_it != sort_index.end()){
- // auto doc_id_val_map = sort_index_it->second;
- // auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
- // if(doc_seq_id_it != doc_id_val_map->end()){
-
- // int64_t doc_val = doc_seq_id_it->second;
- // std::pair<int64_t, std::string> range_pair {};
- // if(a_facet.get_range(doc_val, range_pair)) {
- // int64_t range_id = range_pair.first;
- // facet_count_t& facet_count = a_facet.result_map[range_id];
- // facet_count.count += 1;
- // }
- // }
- // }
- // } else if(!use_facet_query || fquery_doc_id_tokens.find(doc_id) != fquery_doc_id_tokens.end()) {
- // facet_count_t& facet_count = a_facet.result_map[doc_id];
- // //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
- // facet_count.doc_id = doc_seq_id;
- // facet_count.array_pos = j;
- // if(group_limit) {
- // a_facet.hash_groups[doc_id].emplace(distinct_id);
- // } else {
- // facet_count.count += 1;
- // }
- // if(use_facet_query) {
- // a_facet.hash_tokens[doc_id] = fquery_doc_id_tokens.at(doc_id);
- // }
- // }
- // }
- //}
- //}
+ if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+ facet_hash_count = facet_map_it->second.size();
+ } else {
+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
+ single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
+
+ if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+ facet_hash_count = 1;
+ fhash = single_facet_map_it->second;
+ }
+
+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;
+ for(size_t j = 0; j < facet_hash_count; j++) {
+ if(facet_field.is_array()) {
+ fhash = facet_map_it->second.hashes[j];
+ }
+ if(should_compute_stats) {
+ std::string fvalue =
+ facet_index_v4->get_facet_by_count_index(a_facet.field_name, fhash);
+ if(!fvalue.empty()) {
+ compute_facet_stats(a_facet, fvalue, facet_field.type);
+ }
+ }
+ if(a_facet.is_range_query) {
+ auto sort_index_it = sort_index.find(a_facet.field_name);
+ if(sort_index_it != sort_index.end()){
+ auto doc_id_val_map = sort_index_it->second;
+ auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
+ if(doc_seq_id_it != doc_id_val_map->end()){
+
+ std::string doc_val = std::to_string(doc_seq_id_it->second);
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count += 1;
+ }
+ }
+ }
+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
+ std::string fhash_str = std::to_string(fhash);
+ facet_count_t& facet_count = a_facet.result_map[fhash_str];
+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
+ facet_count.doc_id = doc_seq_id;
+ facet_count.array_pos = j;
+ if(group_limit) {
+ a_facet.hash_groups[fhash].emplace(distinct_id);
+ } else {
+ facet_count.count += 1;
+ }
+ if(use_facet_query) {
+ //LOG (INFO) << "adding hash tokens for hash " << fhash;
+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+ }
+ }
+ }
+ }
+ }
+ }
}
void Index::aggregate_topster(Topster* agg_topster, Topster* index_topster) {
@@ -2429,23 +2442,19 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
while (it.valid()) {
uint32_t seq_id = it.id();
uint64_t distinct_id = seq_id;
- // if (group_limit != 0) {
- // distinct_id = get_distinct_id(group_by_fields, seq_id);
- // if(excluded_group_ids.count(distinct_id) != 0) {
- // continue;
- // }
- // }
+ if (group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ if(excluded_group_ids.count(distinct_id) != 0) {
+ continue;
+ }
+ }
int64_t scores[3] = {0};
scores[0] = seq_id;
int64_t match_score_index = -1;
result_ids.push_back(seq_id);
- if(group_limit == 0) {
- KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
- topster->add(&kv);
- }
-
+
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, nullptr);
int ret = topster->add(&kv);
@@ -2541,12 +2550,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
uint64_t distinct_id = seq_id;
- // if (group_limit != 0) {
- // distinct_id = get_distinct_id(group_by_fields, seq_id);
- // if(excluded_group_ids.count(distinct_id) != 0) {
- // continue;
- // }
- // }
+ if (group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ if(excluded_group_ids.count(distinct_id) != 0) {
+ continue;
+ }
+ }
auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :
dist_label.first;
@@ -2564,9 +2573,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores, nullptr);
int ret = topster->add(&kv);
- // if(group_limit != 0 && ret < 2) {
- // groups_processed[distinct_id]++;
- // }
+ if(group_limit != 0 && ret < 2) {
+ groups_processed[distinct_id]++;
+ }
nearest_ids.push_back(seq_id);
}
@@ -2860,8 +2869,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
bool estimate_facets = (facet_sample_percent < 100 && all_result_ids_len > facet_sample_threshold);
if(!facets.empty()) {
- //const size_t num_threads = std::min(concurrency, all_result_ids_len);
- const size_t num_threads = 1;
+ const size_t num_threads = std::min(concurrency, all_result_ids_len);
+ //const size_t num_threads = 1;
const size_t window_size = (num_threads == 0) ? 0 :
(all_result_ids_len + num_threads - 1) / num_threads; // rounds up
size_t num_processed = 0;
@@ -2875,7 +2884,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<std::vector<facet>> facet_batches(num_threads);
for(size_t i = 0; i < num_threads; i++) {
for(const auto& this_facet: facets) {
- facet_batches[i].emplace_back(facet(this_facet.field_name, this_facet.facet_range_map, this_facet.is_range_query));
+ facet_batches[i].emplace_back(facet(this_facet.field_name, this_facet.facet_range_map,
+ this_facet.is_range_query));
}
}
@@ -2930,12 +2940,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto& this_facet = facet_batch[fi];
auto& acc_facet = facets[fi];
for(auto & facet_kv: this_facet.result_map) {
+ uint32_t fhash = 0;
if(group_limit) {
+ fhash = std::stoul(facet_kv.first);
// we have to add all group sets
- // acc_facet.hash_groups[facet_kv.first].insert(
- // this_facet.hash_groups[facet_kv.first].begin(),
- // this_facet.hash_groups[facet_kv.first].end()
- // );
+ acc_facet.hash_groups[fhash].insert(
+ this_facet.hash_groups[fhash].begin(),
+ this_facet.hash_groups[fhash].end()
+ );
} else {
size_t count = 0;
if(acc_facet.result_map.count(facet_kv.first) == 0) {
@@ -2947,10 +2959,14 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
acc_facet.result_map[facet_kv.first].count = count;
}
- //acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
- //acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
- //acc_facet.hash_tokens[facet_kv.first] = this_facet.hash_tokens[facet_kv.first];
- acc_facet.facet_tokens[facet_kv.first] = this_facet.facet_tokens[facet_kv.first];
+ acc_facet.result_map[facet_kv.first].doc_id = facet_kv.second.doc_id;
+ acc_facet.result_map[facet_kv.first].array_pos = facet_kv.second.array_pos;
+ acc_facet.is_intersected = this_facet.is_intersected;
+ if(!facet_query.query.empty()) {
+ fhash = std::stoul(facet_kv.first);
+ //LOG(INFO) << "copying hash tokens for hash " << fhash;
+ acc_facet.hash_tokens[fhash] = this_facet.hash_tokens[fhash];
+ }
}
if(this_facet.stats.fvcount != 0) {
@@ -2964,9 +2980,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
for(auto & acc_facet: facets) {
for(auto& facet_kv: acc_facet.result_map) {
- // if(group_limit) {
- // facet_kv.second.count = acc_facet.hash_groups[facet_kv.first].size();
- // }
+ if(group_limit) {
+ facet_kv.second.count = acc_facet.hash_groups[std::stoul(facet_kv.first)].size();
+ }
if(estimate_facets) {
facet_kv.second.count = size_t(double(facet_kv.second.count) * (100.0f / facet_sample_percent));
@@ -3699,12 +3715,12 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
}
uint64_t distinct_id = seq_id;
- // if(group_limit != 0) {
- // distinct_id = get_distinct_id(group_by_fields, seq_id);
- // if(excluded_group_ids.count(distinct_id) != 0) {
- // return;
- // }
- // }
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ if(excluded_group_ids.count(distinct_id) != 0) {
+ return;
+ }
+ }
int64_t scores[3] = {0};
int64_t match_score_index = -1;
@@ -4301,20 +4317,18 @@ void Index::do_infix_search(const size_t num_search_fields, const std::vector<se
100, scores, match_score_index);
uint64_t distinct_id = seq_id;
- // if(group_limit != 0) {
- // distinct_id = get_distinct_id(group_by_fields, seq_id);
- // if(excluded_group_ids.count(distinct_id) != 0) {
- // continue;
- // }
- // }
- if(group_limit == 0) {
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ if(excluded_group_ids.count(distinct_id) != 0) {
+ continue;
+ }
+ }
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = actual_topster->add(&kv);
- }
- // if(group_limit != 0 && ret < 2) {
- // groups_processed[distinct_id]++;
- // }
+ if(group_limit != 0 && ret < 2) {
+ groups_processed[distinct_id]++;
+ }
if(((i + 1) % (1 << 12)) == 0) {
@@ -4404,13 +4418,13 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
for(size_t findex=0; findex < facets.size(); findex++) {
const auto& a_facet = facets[findex];
- // const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- // const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
+ const auto field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
+ const auto field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
- // if((field_facet_mapping_it == facet_index_v3.end())
- // && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
- // continue;
- // }
+ if((field_facet_mapping_it == facet_index_v3.end())
+ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
+ continue;
+ }
facet_infos[findex].use_facet_query = false;
@@ -4475,16 +4489,6 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
for(auto leaf: searched_query) {
posting_lists.push_back(leaf->values);
std::string tok(reinterpret_cast<char*>(leaf->key), leaf->key_len - 1);
-
- //convert again to boolean string to help search in facet_index map
- if (facet_field.is_bool()) {
- if (tok == "1") {
- tok = "true";
- } else if (tok == "0") {
- tok = "false";
- }
- }
-
searched_tokens.push_back(tok);
//LOG(INFO) << "tok: " << tok;
}
@@ -4513,45 +4517,38 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
}
//LOG(INFO) << "seq_id matched : " << seq_id;
- // if(facet_field.is_array()) {
- // const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- // if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- // continue;
- // }
-
- // std::vector<size_t> array_indices;
- // posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);
-
- // for(size_t array_index: array_indices) {
- // if(array_index < doc_fvalues_it->second.length) {
- // uint64_t hash = doc_fvalues_it->second.hashes[array_index];
-
- // /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "
- // << array_index;*/
-
- // if(facet_infos[findex].hashes.count(hash) == 0) {
- // facet_infos[findex].hashes.emplace(hash, searched_tokens);
- // }
- // }
- // }
- // } else {
- // const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- // if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- // continue;
- // }
+ if(facet_field.is_array()) {
+ const auto doc_fvalues_it = field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(doc_fvalues_it == field_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+
+ std::vector<size_t> array_indices;
+ posting_t::get_matching_array_indices(posting_lists, seq_id, array_indices);
+
+ for(size_t array_index: array_indices) {
+ if(array_index < doc_fvalues_it->second.length) {
+ uint32_t hash = doc_fvalues_it->second.hashes[array_index];
+
+ /*LOG(INFO) << "seq_id: " << seq_id << ", hash: " << hash << ", array index: "
+ << array_index;*/
+
+ if(facet_infos[findex].hashes.count(hash) == 0) {
+ //LOG(INFO) << "adding searched_tokens for hash " << hash;
+ facet_infos[findex].hashes.emplace(hash, searched_tokens);
+ }
+ }
+ }
+ } else {
+ const auto doc_single_fvalues_it = field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(doc_single_fvalues_it == field_single_val_facet_mapping_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
- // uint64_t hash = doc_single_fvalues_it->second;
- // if(facet_infos[findex].hashes.count(hash) == 0) {
- // facet_infos[findex].hashes.emplace(hash, searched_tokens);
- // }
- // }
- }
- std::vector<std::string> matched_facets;
- if(facet_index_v4->get_facet(a_facet.field_name, searched_tokens, matched_facets)) {
- for(const auto& facet : matched_facets) {
- if(facet_infos[findex].facet_tokens.count(facet) == 0) {
- LOG(INFO) << "adding facet " << facet << " in facet_info";
- facet_infos[findex].facet_tokens.emplace(facet, searched_tokens);
+ uint32_t hash = doc_single_fvalues_it->second;
+ if(facet_infos[findex].hashes.count(hash) == 0) {
+ //LOG(INFO) << "adding searched_tokens for hash " << hash;
+ facet_infos[findex].hashes.emplace(hash, searched_tokens);
}
}
}
@@ -4664,20 +4661,18 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
100, scores, match_score_index);
uint64_t distinct_id = seq_id;
- // if(group_limit != 0) {
- // distinct_id = get_distinct_id(group_by_fields, seq_id);
- // if(excluded_group_ids.count(distinct_id) != 0) {
- // continue;
- // }
- // }
- if(group_limit == 0) {
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ if(excluded_group_ids.count(distinct_id) != 0) {
+ continue;
+ }
+ }
KV kv(searched_queries.size(), seq_id, distinct_id, match_score_index, scores);
int ret = topsters[thread_id]->add(&kv);
- }
- // if(group_limit != 0 && ret < 2) {
- // tgroups_processed[thread_id][distinct_id]++;
- // }
+ if(group_limit != 0 && ret < 2) {
+ tgroups_processed[thread_id][distinct_id]++;
+ }
if(check_for_circuit_break && ((i + 1) % (1 << 15)) == 0) {
// check only once every 2^15 docs to reduce overhead
BREAK_CIRCUIT_BREAKER
@@ -5263,16 +5258,12 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
uint64_t distinct_id = seq_id;
- // if(group_limit != 0) {
- // distinct_id = get_distinct_id(group_by_fields, seq_id);
- // groups_processed.emplace(distinct_id);
- // }
+ if(group_limit != 0) {
+ distinct_id = get_distinct_id(group_by_fields, seq_id);
+ groups_processed.emplace(distinct_id);
+ }
//LOG(INFO) << "Seq id: " << seq_id << ", match_score: " << match_score;
- if(group_limit == 0) {
- KV kv(query_index, seq_id, distinct_id, match_score_index, scores);
- topster->add(&kv);
- }
KV kv(query_index, seq_id, distinct_id, match_score_index, scores);
int ret = topster->add(&kv);
if(group_limit != 0 && ret < 2) {
@@ -5286,46 +5277,46 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
// pre-filter group_by_fields such that we can avoid the find() check
uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
const uint32_t seq_id) const {
- uint64_t distinct_id = 1; // some constant initial value
+ uint32_t distinct_id = 1; // some constant initial value
// calculate hash from group_by_fields
for(const auto& field: group_by_fields) {
- // const auto& field_facet_mapping_it = facet_index_v3.find(field);
- // const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);
- // if((field_facet_mapping_it == facet_index_v3.end())
- // && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
- // continue;
- // }
- if(!facet_index_v4->contains(field)) {
+ const auto& field_facet_mapping_it = facet_index_v3.find(field);
+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(field);
+ if((field_facet_mapping_it == facet_index_v3.end())
+ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
continue;
}
+ // if(!facet_index_v4->contains(field)) {
+ // continue;
+ // }
- // if(search_schema.at(field).is_array()) {
- // const auto& field_facet_mapping = field_facet_mapping_it->second;
- // const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
-
- // if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
- // continue;
- // }
-
- // const auto& facet_hashes = facet_hashes_it->second;
+ if(search_schema.at(field).is_array()) {
+ const auto& field_facet_mapping = field_facet_mapping_it->second;
+ const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- // for(size_t i = 0; i < facet_hashes.size(); i++) {
- // distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);
- // }
- // } else {
- // const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- // const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
- // if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
- // continue;
- // }
+ const auto& facet_hashes = facet_hashes_it->second;
+ //LOG(INFO) << "combining hashes for facet array ";
+ for(size_t i = 0; i < facet_hashes.size(); i++) {
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes.hashes[i]);
+ }
+ } else {
+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
+ const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- // const auto& facet_hash = facet_hashes_it->second;
+ if(facet_hashes_it == field_facet_mapping[seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
- // distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
- // }
+ const auto& facet_hash = facet_hashes_it->second;
+ //LOG(INFO) << "combining hashes for facet ";
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
+ }
}
-
+ //LOG(INFO) << "distinct id " << distinct_id;
return distinct_id;
}
@@ -5559,21 +5550,21 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
}
// remove facets
- // const auto& field_facets_it = facet_index_v3.find(field_name);
- // if(field_facets_it != facet_index_v3.end()) {
- // const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- // if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- // field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
- // }
- // }
+ const auto& field_facets_it = facet_index_v3.find(field_name);
+ if(field_facets_it != facet_index_v3.end()) {
+ const auto& fvalues_it = field_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(fvalues_it != field_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
+ field_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
+ }
+ }
- // const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);
- // if(field_single_val_facets_it != single_val_facet_index_v3.end()) {
- // const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
- // if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
- // field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
- // }
- // }
+ const auto& field_single_val_facets_it = single_val_facet_index_v3.find(field_name);
+ if(field_single_val_facets_it != single_val_facet_index_v3.end()) {
+ const auto& fvalues_it = field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->find(seq_id);
+ if(fvalues_it != field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->end()) {
+ field_single_val_facets_it->second[seq_id % ARRAY_FACET_DIM]->erase(fvalues_it);
+ }
+ }
facet_index_v4->erase(field_name);
@@ -5713,7 +5704,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
if(new_field.is_facet()) {
- //initialize_facet_indexes(new_field);
+ initialize_facet_indexes(new_field);
// initialize for non-string facet fields
if(!new_field.is_string()) {
@@ -5777,21 +5768,21 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
}
if(del_field.is_facet()) {
- // if(del_field.is_array()) {
- // auto& arr = facet_index_v3[del_field.name];
- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- // delete arr[i];
- // }
+ if(del_field.is_array()) {
+ auto& arr = facet_index_v3[del_field.name];
+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
+ delete arr[i];
+ }
- // facet_index_v3.erase(del_field.name);
- // } else {
- // auto& arr = single_val_facet_index_v3[del_field.name];
- // for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
- // delete arr[i];
- // }
+ facet_index_v3.erase(del_field.name);
+ } else {
+ auto& arr = single_val_facet_index_v3[del_field.name];
+ for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {
+ delete arr[i];
+ }
- // single_val_facet_index_v3.erase(del_field.name);
- // }
+ single_val_facet_index_v3.erase(del_field.name);
+ }
facet_index_v4->erase(del_field.name);
if(!del_field.is_string()) {
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index ad1cb4db..8827c678 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -70,7 +70,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
std::vector<std::string> facets = {"tags"};
// single facet with no filters
- nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(1, results["facet_counts"].size());
@@ -97,17 +97,18 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1,
FREQUENCY, {false}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 2).get();
+
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_STREQ("tags", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);
+ ASSERT_STREQ("gold", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
+ ASSERT_STREQ("silver", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][1]["count"]);
// 2 facets, 1 text query with no filters
facets.clear();
@@ -230,12 +231,13 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
ASSERT_STREQ("age", results["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("24", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("21", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("21", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>2</mark>1", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("24", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>2</mark>4", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+
// facet on a float field without query to check on stats
results = coll_array_fields->search("*", query_fields, "", {"rating"}, sort_fields, {0}, 10, 1, FREQUENCY,
@@ -615,8 +617,6 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "categories:cell ph").get();
- LOG(INFO) << results.dump();
-
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
@@ -636,8 +636,8 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
- ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cello</mark>phanes", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Cell</mark> Phones", results["facet_counts"][0]["counts"][1]["highlighted"].get<std::string>().c_str());
ASSERT_STREQ("<mark>Cell</mark> Phone Accessories", results["facet_counts"][0]["counts"][2]["highlighted"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
@@ -674,7 +674,6 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) {
1, FREQUENCY, {false});
auto results = res_op.get();
- LOG(INFO) << results.dump();
ASSERT_EQ(7, results["hits"].size());
@@ -1219,14 +1218,20 @@ TEST_F(CollectionFacetingTest, RangeFacetTest) {
ASSERT_TRUE(coll1->add(doc4.dump()).ok());
ASSERT_TRUE(coll1->add(doc5.dump()).ok());
- auto results = coll1->search("Karnataka", {"state"},
+ auto result = coll1->search("Karnataka", {"state"},
"", {"visitors(Busy:[0, 200000], VeryBusy:[200000, 500000])"},
{}, {2}, 10,
1, FREQUENCY, {true},
10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 10, {}, {}, {}, 0,
"<mark>", "</mark>", {}, 1000,
- true, false, true, "", true).get();
+ true, false, true, "", true);
+ if(!result.ok()) {
+ LOG(INFO) << result.error();
+ }
+
+ auto results = result.get();
+
ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);
ASSERT_EQ("Busy", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
@@ -1642,8 +1647,6 @@ TEST_F(CollectionFacetingTest, FacetIndexRefactor) {
// single facet with no filters
nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
- LOG(INFO) << results.dump();
-
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(1, results["facet_counts"].size());
ASSERT_EQ(4, results["facet_counts"][0].size());
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index c094f9fc..f88d9551 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -69,8 +69,6 @@ TEST_F(CollectionGroupingTest, GroupingBasics) {
"", 10,
{}, {}, {"size"}, 2).get();
- LOG(INFO) << res.dump();
-
ASSERT_EQ(3, res["found"].get<size_t>());
ASSERT_EQ(3, res["grouped_hits"].size());
ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
@@ -454,10 +452,10 @@ TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);
- ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
-
+ ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
+
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);
- ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index ab0e951e..e01e7b27 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2384,7 +2384,7 @@ TEST_F(CollectionTest, UpdateDocument) {
res = coll1->search("lazy", {"title"}, "", {"tags"}, sort_fields, {0}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "title").get();
-
+
ASSERT_EQ(1, res["hits"].size());
ASSERT_EQ(1, res["facet_counts"].size());
ASSERT_STREQ("tags", res["facet_counts"][0]["field_name"].get<std::string>().c_str());