Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

revert old changes, test changes for wildcard

krunale107ddfa9396
diff --git a/include/facet_index.h b/include/facet_index.h
index 0c681a3a..33ad924e 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -82,16 +82,14 @@ private:
std::list<facet_count_t> counts;
posting_list_t* seq_id_hashes = nullptr;
spp::sparse_hash_map<uint32_t, int64_t> fhash_to_int64_map;
- posting_list_t::iterator_t facet_index_it;
bool has_value_index = true;
bool has_hash_index = true;
- facet_doc_ids_list_t() :
- seq_id_hashes(new posting_list_t(256)),
- facet_index_it(seq_id_hashes->new_iterator()) {
+ facet_doc_ids_list_t() {
fvalue_seq_ids.clear();
counts.clear();
+ seq_id_hashes = new posting_list_t(256);
}
facet_doc_ids_list_t(const facet_doc_ids_list_t& other) = delete;
@@ -154,8 +152,6 @@ public:
posting_list_t* get_facet_hash_index(const std::string& field_name);
- posting_list_t::iterator_t* get_facet_index_it(const std::string& field_name);
-
//get fhash=>int64 map for stats
const spp::sparse_hash_map<uint32_t, int64_t>& get_fhash_int64_map(const std::string& field_name);
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index a78cf783..fa9eeef1 100644
--- a/include/index.h
+++ b/include/index.h
@@ -623,6 +623,9 @@ public:
uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields,
const uint32_t seq_id, const bool group_missing_values) const;
+ void get_distinct_id(const std::string& field_name, posting_list_t::iterator_t& facet_index_it,
+ const uint32_t seq_id, const bool group_missing_values, uint64_t& distinct_id) const;
+
static void compute_token_offsets_facets(index_record& record,
const tsl::htrie_map<char, field>& search_schema,
const std::vector<char>& local_token_separators,
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index 7ab11481..ff519fe5 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -344,14 +344,6 @@ posting_list_t* facet_index_t::get_facet_hash_index(const std::string &field_nam
return nullptr;
}
-posting_list_t::iterator_t* facet_index_t::get_facet_index_it(const std::string& field_name) {
- auto facet_index_it = facet_field_map.find(field_name);
- if(facet_index_it != facet_field_map.end()) {
- return &facet_index_it->second.facet_index_it;
- }
- return nullptr;
-}
-
const spp::sparse_hash_map<uint32_t , int64_t >& facet_index_t::get_fhash_int64_map(const std::string& field_name) {
const auto facet_field_map_it = facet_field_map.find(field_name);
if(facet_field_map_it == facet_field_map.end()) {
diff --git a/src/index.cpp b/src/index.cpp
index 0d6fe805..333dac17 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -5223,7 +5223,17 @@ Option<bool> Index::search_wildcard(filter_node_t const* const& filter_tree_root
uint64_t distinct_id = seq_id;
if(group_limit != 0) {
- distinct_id = get_distinct_id(group_by_fields, seq_id, group_missing_values);
+ distinct_id = 1;
+ for(const auto& field_name: group_by_fields) {
+ if (!facet_index_v4->has_hash_index(field_name)) {
+ continue;
+ }
+ auto facet_index = facet_index_v4->get_facet_hash_index(field_name);
+ auto facet_index_it = facet_index->new_iterator();
+
+ get_distinct_id(field_name, facet_index_it, seq_id, group_missing_values, distinct_id);
+ }
+
if(excluded_group_ids.count(distinct_id) != 0) {
continue;
}
@@ -5890,28 +5900,27 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
// calculate hash from group_by_fields
for(const auto& field_name: group_by_fields) {
- if(!facet_index_v4->has_hash_index(field_name)) {
+ if (!facet_index_v4->has_hash_index(field_name)) {
continue;
}
std::vector<uint32_t> facet_hashes;
- posting_list_t::iterator_t* facet_index_it = facet_index_v4->get_facet_index_it(field_name);
- if(facet_index_it) {
- facet_index_it->skip_to(seq_id);
+ auto facet_index = facet_index_v4->get_facet_hash_index(field_name);
+ auto facet_index_it = facet_index->new_iterator();
+ facet_index_it.skip_to(seq_id);
- if (facet_index_it->valid() && facet_index_it->id() == seq_id) {
- posting_list_t::get_offsets(*facet_index_it, facet_hashes);
+ if (facet_index_it.valid() && facet_index_it.id() == seq_id) {
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
- if (search_schema.at(field_name).is_array()) {
- //LOG(INFO) << "combining hashes for facet array ";
- for (size_t i = 0; i < facet_hashes.size(); i++) {
- distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);
- }
- } else {
- const auto &facet_hash = facet_hashes[0];
- //LOG(INFO) << "combining hashes for facet ";
- distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
+ if (search_schema.at(field_name).is_array()) {
+ //LOG(INFO) << "combining hashes for facet array ";
+ for (size_t i = 0; i < facet_hashes.size(); i++) {
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);
}
+ } else {
+ const auto &facet_hash = facet_hashes[0];
+ //LOG(INFO) << "combining hashes for facet ";
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
}
}
}
@@ -5920,6 +5929,36 @@ uint64_t Index::get_distinct_id(const std::vector<std::string>& group_by_fields,
return (distinct_id == 1 && !group_missing_values) ? seq_id : distinct_id;
}
+void Index::get_distinct_id(const std::string& field_name, posting_list_t::iterator_t& facet_index_it,
+ const uint32_t seq_id, const bool group_missing_values, uint64_t& distinct_id) const {
+
+ // calculate hash from group_by_fields
+ std::vector<uint32_t> facet_hashes;
+ facet_index_it.skip_to(seq_id);
+
+ if (facet_index_it.valid() && facet_index_it.id() == seq_id) {
+ posting_list_t::get_offsets(facet_index_it, facet_hashes);
+
+ if (search_schema.at(field_name).is_array()) {
+ //LOG(INFO) << "combining hashes for facet array ";
+ for (size_t i = 0; i < facet_hashes.size(); i++) {
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hashes[i]);
+ }
+ } else {
+ const auto &facet_hash = facet_hashes[0];
+ //LOG(INFO) << "combining hashes for facet ";
+ distinct_id = StringUtils::hash_combine(distinct_id, facet_hash);
+ }
+ }
+
+ //LOG(INFO) << "seq_id: " << seq_id << ", distinct_id: " << distinct_id;
+ if (distinct_id == 1 && !group_missing_values) {
+ distinct_id = seq_id;
+ }
+
+ return;
+}
+
inline uint32_t Index::next_suggestion2(const std::vector<tok_candidates>& token_candidates_vec,
long long int n,
std::vector<token_t>& query_suggestion,