Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.25-join' of https://github.com/typesense/typesense into facet_index_refactor

krunal1313c6a2c7946bad
diff --cc include/field.h
index 9e8b32a2,4fa6cc6e..696c0bb0
--- a/include/field.h
+++ b/include/field.h
@@@ -715,9 -833,9 +832,11 @@@ struct facet
bool sampled = false;
+ bool is_wildcard_match = false;
+
- bool get_range(int64_t key, std::pair<int64_t, std::string>& range_pair)
+ bool is_intersected = false;
+
+ bool get_range(std::string key, std::pair<std::string, std::string>& range_pair)
{
if(facet_range_map.empty())
{
@@@ -765,12 -882,11 +883,10 @@@ struct facet_value_t
struct facet_hash_values_t {
uint32_t length = 0;
- //uint32_t* hashes = nullptr;
- uint64_t* hashes = nullptr;
+ std::vector<uint32_t> hashes;
facet_hash_values_t() {
length = 0;
- //hashes = nullptr;
- hashes = nullptr;
}
facet_hash_values_t(facet_hash_values_t&& hash_values) noexcept {
@@@ -778,20 -894,17 +894,17 @@@
hashes = hash_values.hashes;
hash_values.length = 0;
- //hash_values.hashes = nullptr;
- hash_values.hashes = nullptr;
+ hash_values.hashes.clear();
}
facet_hash_values_t& operator=(facet_hash_values_t&& other) noexcept {
if (this != &other) {
- //delete[] hashes;
- delete[] hashes;
+ hashes.clear();
hashes = other.hashes;
length = other.length;
- //other.hashes = nullptr;
- other.hashes = nullptr;
+ other.hashes.clear();
other.length = 0;
}
@@@ -799,9 -912,8 +912,7 @@@
}
~facet_hash_values_t() {
- //delete [] hashes;
- //hashes = nullptr;
- delete [] hashes;
- hashes = nullptr;
+ hashes.clear();
}
uint64_t size() const {
@@@ -809,7 -921,6 +920,6 @@@
}
uint64_t back() const {
- //return hashes[length - 1];
- return hashes[length - 1];
+ return hashes.back();
}
-};
+};
diff --cc include/index.h
index 3cc57b42,332e9fa2..b2b1f8c4
--- a/include/index.h
+++ b/include/index.h
@@@ -187,24 -187,9 +188,9 @@@ struct search_args
};
};
- enum index_operation_t {
- CREATE,
- UPSERT,
- UPDATE,
- EMPLACE,
- DELETE
- };
-
- enum class DIRTY_VALUES {
- REJECT = 1,
- DROP = 2,
- COERCE_OR_REJECT = 3,
- COERCE_OR_DROP = 4,
- };
-
struct offsets_facet_hashes_t {
std::unordered_map<std::string, std::vector<uint32_t>> offsets;
- std::vector<uint64_t> facet_hashes;
+ //std::vector<uint32_t> facet_hashes;
};
struct index_record {
@@@ -514,23 -506,23 +510,19 @@@ private
static void tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
- std::vector<uint32_t>& facet_hashes*/);
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes);
++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets);
static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
- std::vector<uint32_t>& facet_hashes*/);
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes);
++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets);
void collate_included_ids(const std::vector<token_t>& q_included_tokens,
const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,
Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries) const;
-- static uint64_t facet_token_hash(const field & a_field, const std::string &token);
--
- static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
+ static void compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type);
static void get_doc_changes(const index_operation_t op, nlohmann::json &update_doc,
const nlohmann::json &old_doc, nlohmann::json &new_doc, nlohmann::json &del_doc);
diff --cc src/collection.cpp
index dd8ccbc7,90ca8b1a..19aa663a
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1842,129 -1932,104 +1926,129 @@@ Option<nlohmann::json> Collection::sear
LOG (ERROR) << "range_id not found in result map.";
}
}
- }
-
- auto the_field = search_schema.at(a_facet.field_name);
- // keep only top K facets
- auto max_facets = std::min(max_facet_values, facet_hash_counts.size());
- std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
- facet_hash_counts.end(), Collection::facet_count_compare);
- for(size_t fi = 0; fi < max_facets; fi++) {
-
- if(a_facet.is_range_query){
- break;
+ } else if(a_facet.is_intersected) {
- LOG(INFO) << "used intersection";
++ //LOG(INFO) << "used intersection";
+ std::vector<std::pair<std::string, uint32_t>> facet_counts;
+
+ for (const auto & kv : a_facet.result_map) {
+ facet_counts.emplace_back(std::make_pair(kv.first, kv.second.count));
}
+
+ auto max_facets = std::min(max_facet_values, facet_counts.size());
+ std::sort(facet_counts.begin(), facet_counts.end(),
+ [&](const auto& p1, const auto& p2) {
+ return p1.second > p2.second;
+ });
- // remap facet value hash with actual string
- auto & kv = facet_hash_counts[fi];
- auto & facet_count = kv.second;
- // fetch actual facet value from representative doc id
- const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);
- nlohmann::json document;
- const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
- if(!document_op.ok()) {
- LOG(ERROR) << "Facet fetch error. " << document_op.error();
- continue;
+ for(int i = 0; i < max_facets; ++i) {
+ const auto& kv = facet_counts[i];
+ facet_value_t facet_value = { kv.first, kv.first, kv.second};
+ facet_values.emplace_back(facet_value);
}
- std::string value;
- bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);
- if(!facet_found) {
- continue;
+ } else {
- LOG(INFO) << "used hashes";
++ //LOG(INFO) << "used hashes";
+ std::vector<std::pair<uint32_t, facet_count_t>> facet_hash_counts;
+
+ for (const auto & kv : a_facet.result_map) {
+ facet_hash_counts.emplace_back(std::make_pair(std::stoul(kv.first), kv.second));
}
- std::unordered_map<std::string, size_t> ftoken_pos;
- std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];
- for(size_t ti = 0; ti < ftokens.size(); ti++) {
- if(the_field.is_bool()) {
- if(ftokens[ti] == "1") {
- ftokens[ti] = "true";
- } else {
- ftokens[ti] = "false";
- }
+
+ auto the_field = search_schema.at(a_facet.field_name);
+ // keep only top K facets
+ auto max_facets = std::min(max_facet_values, facet_hash_counts.size());
+
+ std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
+ facet_hash_counts.end(), Collection::facet_count_compare);
+
+ for(size_t fi = 0; fi < max_facets; fi++) {
+ // remap facet value hash with actual string
+ auto & kv = facet_hash_counts[fi];
+ auto & facet_count = kv.second;
+ // fetch actual facet value from representative doc id
+ const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);
+ nlohmann::json document;
+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);
+ if(!document_op.ok()) {
+ LOG(ERROR) << "Facet fetch error. " << document_op.error();
+ continue;
}
- const std::string& resolved_token = ftokens[ti];
- ftoken_pos[resolved_token] = ti;
- }
- const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();
- // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text
- bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);
- bool normalise = is_cyrillic ? false : true;
- Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);
- // secondary tokenizer used for specific languages that requires transliteration
- // we use 2 tokenizers so that the original text offsets are available for highlighting
- Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);
- std::string raw_token;
- size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
- // need an ordered map here to ensure that it is ordered by the key (start offset)
- std::map<size_t, size_t> token_offsets;
- size_t prefix_token_start_index = 0;
- while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
- if(is_cyrillic) {
- word_tokenizer.tokenize(raw_token);
+ std::string value;
+ bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);
+ if(!facet_found) {
+ continue;
}
- auto token_pos_it = ftoken_pos.find(raw_token);
- if(token_pos_it != ftoken_pos.end()) {
- token_offsets[tok_start] = tok_end;
- if(raw_token == last_full_q_token) {
- prefix_token_start_index = tok_start;
+ std::unordered_map<std::string, size_t> ftoken_pos;
+ std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];
+ //LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();
+ for(size_t ti = 0; ti < ftokens.size(); ti++) {
+ if(the_field.is_bool()) {
+ if(ftokens[ti] == "1") {
+ ftokens[ti] = "true";
+ } else {
+ ftokens[ti] = "false";
+ }
}
+ const std::string& resolved_token = ftokens[ti];
+ ftoken_pos[resolved_token] = ti;
}
- }
- auto offset_it = token_offsets.begin();
- size_t i = 0;
- std::stringstream highlightedss;
- // loop until end index, accumulate token and complete highlighting
- while(i < value.size()) {
- if(offset_it != token_offsets.end()) {
- if (i == offset_it->first) {
- highlightedss << highlight_start_tag;
- // do prefix highlighting for non-dropped last token
- size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?
- facet_query_last_token.size() :
- (offset_it->second - i + 1);
- if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {
- token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());
- } else {
- token_len = (offset_it->second - i + 1);
+ const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();
+ // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text
+ bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);
+ bool normalise = is_cyrillic ? false : true;
+ Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);
+ // secondary tokenizer used for specific languages that requires transliteration
+ // we use 2 tokenizers so that the original text offsets are available for highlighting
+ Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);
+ std::string raw_token;
+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
+ // need an ordered map here to ensure that it is ordered by the key (start offset)
+ std::map<size_t, size_t> token_offsets;
+ size_t prefix_token_start_index = 0;
+ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
+ if(is_cyrillic) {
+ word_tokenizer.tokenize(raw_token);
+ }
+ auto token_pos_it = ftoken_pos.find(raw_token);
+ if(token_pos_it != ftoken_pos.end()) {
+ token_offsets[tok_start] = tok_end;
+ if(raw_token == last_full_q_token) {
+ prefix_token_start_index = tok_start;
}
- for(size_t j = 0; j < token_len; j++) {
- highlightedss << value[i + j];
+ }
+ }
+ //LOG(INFO) << "token_offsets size " << token_offsets.size();
+ auto offset_it = token_offsets.begin();
+ size_t i = 0;
+ std::stringstream highlightedss;
+ // loop until end index, accumulate token and complete highlighting
+ while(i < value.size()) {
+ if(offset_it != token_offsets.end()) {
+ if (i == offset_it->first) {
+ highlightedss << highlight_start_tag;
+ // do prefix highlighting for non-dropped last token
+ size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?
+ facet_query_last_token.size() :
+ (offset_it->second - i + 1);
+ if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {
+ token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());
+ } else {
+ token_len = (offset_it->second - i + 1);
+ }
+ for(size_t j = 0; j < token_len; j++) {
+ highlightedss << value[i + j];
+ }
+ highlightedss << highlight_end_tag;
+ offset_it++;
+ i += token_len;
+ continue;
}
- highlightedss << highlight_end_tag;
- offset_it++;
- i += token_len;
- continue;
}
+ highlightedss << value[i];
+ i++;
}
- highlightedss << value[i];
- i++;
+ facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};
+ facet_values.emplace_back(facet_value);
}
- facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};
- facet_values.emplace_back(facet_value);
}
std::stable_sort(facet_values.begin(), facet_values.end(), Collection::facet_count_str_compare);
diff --cc src/facet_index.cpp
index 7875f091,00000000..3a8902da
mode 100644,000000..100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@@ -1,159 -1,0 +1,160 @@@
+#include "facet_index.h"
+#include <tokenizer.h>
+#include "string_utils.h"
+
+uint32_t facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {
+ auto& facet_index_map = facet_field_map[field].facet_index_map;
+ uint32_t index;
+ const auto sv = value.substr(0, 100);
- if(facet_index_map.count(sv) == 0) {
++ const auto it = facet_index_map.find(sv);
++ if(it == facet_index_map.end()) {
+ index = ++count_index;
+
+ facet_index_struct fis {};
+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));
+ fis.index = index;
+ facet_index_map.emplace(sv, fis);
+ }else {
+ auto ids = facet_index_map[sv].id_list_ptr;
+ if (!ids_t::contains(ids, id)) {
+ ids_t::upsert(ids, id);
+ facet_index_map[sv].id_list_ptr = ids;
+ }
+ index = facet_index_map[sv].index;
+ }
+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
+ auto& counter_list = facet_field_map[field].counter_list;
+ count_list node(sv, facet_count);
+
+ if(counter_list.empty()) {
+ counter_list.emplace_back(count_list(sv, facet_count));
+ } else {
+ auto it = counter_list.begin();
+ //remove node from list
+ for(it = counter_list.begin(); it != counter_list.end(); ++it) {
+ if(it->facet_value == sv) {
+ //found facet in first node
+ counter_list.erase(it);
+ break;
+ }
+ }
+
+ //find position in list and add node with updated count
+ count_list node(sv, facet_count);
+
+ for(it = counter_list.begin(); it != counter_list.end(); ++it) {
+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value
+ // << " with count " << node.count;
+ if(it->count <= facet_count) {
+ counter_list.emplace(it, node);
+ break;
+ }
+ }
+ if(it == counter_list.end()) {
+ // LOG (INFO) << "inserting at last facet " << node.facet_value
+ // << " with count " << node.count;
+ counter_list.emplace_back(node);
+ }
+ }
+
+ return index;
+}
+
+bool facet_index_t::contains(const std::string& field) {
+
+ const auto& facet_field_it = facet_field_map.find(field);
+ if(facet_field_it == facet_field_map.end()) {
+ return false;
+ }
+
+ return true;
+}
+
+void facet_index_t::erase(const std::string& field) {
+ const auto it = facet_field_map.find(field);
+ if(it != facet_field_map.end()) {
+ facet_field_map.erase(field);
+ }
+}
+
+size_t facet_index_t::get_facet_count(const std::string& field) {
+ const auto it = facet_field_map.find(field);
+
+ if(it == facet_field_map.end()) {
+ return 0;
+ }
+
+ return it->second.counter_list.size();
+}
+
+//returns the count of matching seq_ids from result array
+int facet_index_t::intersect(const std::string& field, const uint32_t* result_ids,
+ int result_ids_len, int max_facet_count,
+ std::map<std::string, uint32_t>& found, bool is_wildcard_no_filter_query) {
+ //LOG (INFO) << "intersecting field " << field;
+
+ const auto& facet_field_it = facet_field_map.find(field);
+ if(facet_field_it == facet_field_map.end()) {
+ return 0;
+ }
+
+ const auto facet_index_map = facet_field_it->second.facet_index_map;
+ const auto counter_list = facet_field_it->second.counter_list;
+
+ // LOG (INFO) << "facet_index_map size " << facet_index_map.size()
+ // << " , counter_list size " << counter_list.size();
+
+ std::vector<uint32_t> id_list;
+ for(const auto& counter_list_it : counter_list) {
+ // LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value
+ // << " having total count " << counter_list_it.count;
+ int count = 0;
+
+ if(is_wildcard_no_filter_query) {
+ count = counter_list_it.count;
+ } else {
+ auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;
+ ids_t::uncompress(ids, id_list);
+ const auto ids_len = id_list.size();
+ for(int i = 0; i < result_ids_len; ++i) {
+ if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {
+ ++count;
+ }
+ }
+ id_list.clear();
+ }
+
+ if(count) {
+ found[counter_list_it.facet_value] = count;
+ if(found.size() == max_facet_count) {
+ break;
+ }
+ }
+ }
+
+ return found.size();
+}
+
+std::string facet_index_t::get_facet_by_count_index(const std::string& field, uint32_t count_index) {
+
+ const auto& facet_field_it = facet_field_map.find(field);
+
+ if(facet_field_it == facet_field_map.end()) {
+ return "";
+ }
+ std::string result = "";
+ auto facet_index_map = facet_field_it->second.facet_index_map;
+
+ for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {
+ if(it.value().index == count_index) {
+ result = it.key();
+ }
+ }
+ return result;
+}
+
+facet_index_t::~facet_index_t() {
+ facet_field_map.clear();
+}
+
diff --cc src/index.cpp
index 29f8e4fd,a24e6cb7..cd7e7abd
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -814,97 -651,30 +656,87 @@@ void Index::index_field_in_memory(cons
if(afield.facet) {
if(afield.is_array()) {
- facet_hash_values_t fhashvalues;
- fhashvalues.length = field_index_it->second.facet_hashes.size();
- fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];
-
- for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {
- fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];
+ facet_hash_values_t fhashvalues;
+
+ if(afield.type == field_types::INT32_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ int32_t raw_val = document[afield.name][i].get<int32_t>();
+ value = std::to_string(raw_val);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
}
-
- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- } else {
- facet_dim_index->emplace(seq_id, std::move(fhashvalues));
+ else if(afield.type == field_types::INT64_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ int64_t raw_val = document[afield.name][i].get<int64_t>();
+ value = std::to_string(raw_val);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ else if(afield.type == field_types::STRING_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ value = document[afield.name][i];
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ else if(afield.type == field_types::FLOAT_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ float raw_val = document[afield.name][i].get<float>();
+ value = StringUtils::float_to_str(raw_val);
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
+ }
+ else if(afield.type == field_types::BOOL_ARRAY) {
+ for(int i = 0; i < document[afield.name].size(); ++i) {
+ value = std::to_string(document[afield.name][i].get<bool>());
- //value = (value == "1") ? "true" : "false";
+ auto index = facet_index_v4->insert(afield.name, value, seq_id);
+ fhashvalues.hashes.emplace_back(index);
+ }
}
+ fhashvalues.length = fhashvalues.hashes.size();
- //LOG(INFO) << "fhashvalues.length " << fhashvalues.length;
- //fhashvalues.length = field_index_it->second.facet_hashes.size();
- //fhashvalues.hashes = new uint32_t[field_index_it->second.facet_hashes.size()];
-
- // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {
- // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];
- // }
+
+ auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
+ } else {
+ facet_dim_index->emplace(seq_id, std::move(fhashvalues));
+ }
} else {
- uint64_t fhash;
- fhash = field_index_it->second.facet_hashes[0];
-
- auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
- if(facet_dim_index == nullptr) {
- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
- } else {
- facet_dim_index->emplace(seq_id, std::move(fhash));
+ uint32_t fhash;
+
+ if(afield.type == field_types::INT32) {
+ int32_t raw_val = document[afield.name].get<int32_t>();
+ value = std::to_string(raw_val);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ }
+ else if(afield.type == field_types::INT64) {
+ int64_t raw_val = document[afield.name].get<int64_t>();
+ value = std::to_string(raw_val);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
}
+ else if(afield.type == field_types::STRING) {
+ value = document[afield.name];
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ }
+ else if(afield.type == field_types::FLOAT) {
+ float raw_val = document[afield.name].get<float>();
+ value = StringUtils::float_to_str(raw_val);
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ }
+ else if(afield.type == field_types::BOOL) {
+ value = std::to_string(document[afield.name].get<bool>());
- //value = (value == "1") ? "true" : "false";
+ fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ }
- //fhash = field_index_it->second.facet_hashes[0];
+
+ auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];
+ if(facet_dim_index == nullptr) {
+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;
+ } else {
+ facet_dim_index->emplace(seq_id, std::move(fhash));
+ }
}
}
@@@ -1185,68 -966,73 +1028,40 @@@
}
}
--uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {
-- // for integer/float use their native values
-- uint64_t hash = 0;
--
-- if(a_field.is_float()) {
-- float f = std::stof(token);
-- reinterpret_cast<float&>(hash) = f; // store as int without loss of precision
-- } else if(a_field.is_integer() || a_field.is_bool()) {
-- hash = atoll(token.c_str());
-- } else {
-- // string field
-- hash = StringUtils::hash_wy(token.c_str(), token.size());
-- }
--
-- return hash;
--}
--
void Index::tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
- std::vector<uint32_t>& facet_hashes*/) {
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes) {
++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets) {
Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
std::string token;
std::string last_token;
size_t token_index = 0;
-- uint64_t facet_hash = 1;
while(tokenizer.next(token, token_index)) {
if(token.empty()) {
continue;
}
+ if(token.size() > 100) {
+ token.erase(100);
+ }
+
token_to_offsets[token].push_back(token_index + 1);
last_token = token;
- // if(is_facet) {
- // uint64_t token_hash = Index::facet_token_hash(a_field, token);
- // if(token_index == 0) {
- // facet_hash = token_hash;
- // } else {
- // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
- // }
- // }
-
- if(is_facet) {
- uint64_t token_hash = Index::facet_token_hash(a_field, token);
- if(token_index == 0) {
- facet_hash = token_hash;
- } else {
- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
- }
- }
}
if(!token_to_offsets.empty()) {
// push 0 for the last occurring token (used for exact match ranking)
token_to_offsets[last_token].push_back(0);
}
--
- // if(is_facet) {
- // facet_hashes.push_back(facet_hash);
- // }
- if(is_facet) {
- facet_hashes.push_back(facet_hash);
- }
}
void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,
const field& a_field,
const std::vector<char>& symbols_to_index,
const std::vector<char>& token_separators,
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,
- std::vector<uint32_t>& facet_hashes*/) {
- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,
- std::vector<uint64_t>& facet_hashes) {
++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets) {
for(size_t array_index = 0; array_index < strings.size(); array_index++) {
const std::string& str = strings[array_index];
@@@ -1255,7 -1041,7 +1070,6 @@@
Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);
std::string token, last_token;
size_t token_index = 0;
-- uint64_t facet_hash = 1;
// iterate and append offset positions
while(tokenizer.next(token, token_index)) {
@@@ -1266,21 -1056,21 +1084,8 @@@
token_to_offsets[token].push_back(token_index + 1);
token_set.insert(token);
last_token = token;
--
- // if(is_facet) {
- // uint64_t token_hash = Index::facet_token_hash(a_field, token);
- // if(token_index == 0) {
- // facet_hash = token_hash;
- // } else {
- // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
- // }
- // }
- if(is_facet) {
- uint64_t token_hash = Index::facet_token_hash(a_field, token);
- if(token_index == 0) {
- facet_hash = token_hash;
- } else {
- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);
- }
- }
- }
-
- if(is_facet) {
- facet_hashes.push_back(facet_hash);
}
- // if(is_facet) {
- // facet_hashes.push_back(facet_hash);
- // }
-
if(token_set.empty()) {
continue;
}
@@@ -1363,120 -1153,91 +1168,129 @@@ void Index::do_facets(std::vector<facet
size_t mod_value = 100 / facet_sample_percent;
- facet_map_t::iterator facet_map_it;
- single_val_facet_map_t::iterator single_facet_map_it;
- uint64_t fhash = 0;
- size_t facet_hash_count = 1;
- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
+ const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);
- if((field_facet_mapping_it == facet_index_v3.end())
- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
- continue;
- }
+ if(results_size && facet_records && (facet_records <= 10 || is_wildcard_query == true)
+ && use_facet_query == false && group_limit == 0 && no_filters_provided == true) {
+ //LOG(INFO) << "Using intersection to find facets";
+ a_facet.is_intersected = true;
- for(size_t i = 0; i < results_size; i++) {
- // if sampling is enabled, we will skip a portion of the results to speed up things
- if(estimate_facets) {
- if(i % mod_value != 0) {
- continue;
- }
- }
-
- uint32_t doc_seq_id = result_ids[i];
- if(facet_field.is_array()) {
- const auto& field_facet_mapping = field_facet_mapping_it->second;
- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
- continue;
+ std::map<std::string, uint32_t> facet_results;
+ facet_index_v4->intersect(a_facet.field_name, result_ids,
+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);
+
+ for(const auto& kv : facet_results) {
+ //range facet processing
+ if(a_facet.is_range_query) {
+ const auto doc_val = kv.first;
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count = kv.second;
+ }
+ } else {
+ facet_count_t& facet_count = a_facet.result_map[kv.first];
+ facet_count.count = kv.second;
}
- facet_hash_count = facet_map_it->second.size();
- } else {
- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
-
- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+
+ if(should_compute_stats) {
+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;
+ for(int i = 0; i < kv.second; ++i) {
+ compute_facet_stats(a_facet, kv.first, facet_field.type);
+ }
+ }
+ }
+ } else {
+ facet_map_t::iterator facet_map_it;
+ single_val_facet_map_t::iterator single_facet_map_it;
+ uint32_t fhash = 0;
+ size_t facet_hash_count = 1;
+
+ const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);
+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);
+
++ if((field_facet_mapping_it == facet_index_v3.end())
++ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
+ continue;
- }
- facet_hash_count = 1;
- fhash = single_facet_map_it->second;
+ }
-
- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
+
- if(((i + 1) % 16384) == 0) {
- RETURN_CIRCUIT_BREAKER
- }
+ for(size_t i = 0; i < results_size; i++) {
+ // if sampling is enabled, we will skip a portion of the results to speed up things
+ if(estimate_facets) {
+ if(i % mod_value != 0) {
+ continue;
+ }
+ }
- for(size_t j = 0; j < facet_hash_count; j++) {
-
+ uint32_t doc_seq_id = result_ids[i];
if(facet_field.is_array()) {
- fhash = facet_map_it->second.hashes[j];
+ const auto& field_facet_mapping = field_facet_mapping_it->second;
+ facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
+
+ if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+ facet_hash_count = facet_map_it->second.size();
+ } else {
+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;
+ single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);
+
+ if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {
+ continue;
+ }
+ facet_hash_count = 1;
+ fhash = single_facet_map_it->second;
}
-
- if(should_compute_stats) {
- compute_facet_stats(a_facet, fhash, facet_field.type);
+
+ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;
+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;
++ if(((i + 1) % 16384) == 0) {
++ RETURN_CIRCUIT_BREAKER
+ }
- if(a_facet.is_range_query) {
- auto sort_index_it = sort_index.find(a_facet.field_name);
-
- if(sort_index_it != sort_index.end()){
- auto doc_id_val_map = sort_index_it->second;
- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
-
- if(doc_seq_id_it != doc_id_val_map->end()){
-
- int64_t doc_val = doc_seq_id_it->second;
- std::pair<int64_t, std::string> range_pair {};
- if(a_facet.get_range(doc_val, range_pair)) {
- int64_t range_id = range_pair.first;
- facet_count_t& facet_count = a_facet.result_map[range_id];
- facet_count.count += 1;
- }
- }
++
+ for(size_t j = 0; j < facet_hash_count; j++) {
+ if(facet_field.is_array()) {
+ fhash = facet_map_it->second.hashes[j];
}
- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
- facet_count_t& facet_count = a_facet.result_map[fhash];
- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
- facet_count.doc_id = doc_seq_id;
- facet_count.array_pos = j;
- if(group_limit) {
- a_facet.hash_groups[fhash].emplace(distinct_id);
- } else {
- facet_count.count += 1;
+ if(should_compute_stats) {
+ std::string fvalue =
+ facet_index_v4->get_facet_by_count_index(a_facet.field_name, fhash);
+ if(!fvalue.empty()) {
+ compute_facet_stats(a_facet, fvalue, facet_field.type);
+ }
}
- if(use_facet_query) {
- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+ if(a_facet.is_range_query) {
+ auto sort_index_it = sort_index.find(a_facet.field_name);
+ if(sort_index_it != sort_index.end()){
+ auto doc_id_val_map = sort_index_it->second;
+ auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);
+ if(doc_seq_id_it != doc_id_val_map->end()){
+
+ std::string doc_val = std::to_string(doc_seq_id_it->second);
+ std::pair<std::string, std::string> range_pair {};
+ if(a_facet.get_range(doc_val, range_pair)) {
+ const auto& range_id = range_pair.first;
+ facet_count_t& facet_count = a_facet.result_map[range_id];
+ facet_count.count += 1;
+ }
+ }
+ }
+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {
+ std::string fhash_str = std::to_string(fhash);
+ facet_count_t& facet_count = a_facet.result_map[fhash_str];
+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;
+ facet_count.doc_id = doc_seq_id;
+ facet_count.array_pos = j;
+ if(group_limit) {
+ a_facet.hash_groups[fhash].emplace(distinct_id);
+ } else {
+ facet_count.count += 1;
+ }
+ if(use_facet_query) {
+ //LOG (INFO) << "adding hash tokens for hash " << fhash;
+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);
+ }
}
}
}
@@@ -1659,7 -1421,7 +1474,6 @@@ void Index::search_candidates(const uin
);
// We fetch offset positions only for multi token query
-- bool fetch_offsets = (query_suggestion.size() > 1);
bool single_exact_query_token = false;
if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {
@@@ -1856,7 -1706,7 +1758,6 @@@ Option<bool> Index::do_filtering(filter
if (is_polygon) {
const int num_verts = int(filter_value_parts.size()) / 2;
std::vector<S2Point> vertices;
-- double sum = 0.0;
for (size_t point_index = 0; point_index < size_t(num_verts);
point_index++) {
@@@ -2515,11 -2453,11 +2505,8 @@@ void Index::process_filter_overrides(co
// we will cover both original query and synonyms
std::vector<std::string> rule_parts;
- StringUtils::split(override->rule.query, rule_parts, " ");
-
- uint32_t* field_override_ids = nullptr;
- size_t field_override_ids_len = 0;
+ StringUtils::split(override->rule.normalized_query, rule_parts, " ");
- uint32_t* field_override_ids = nullptr;
- size_t field_override_ids_len = 0;
-
bool exact_rule_match = override->rule.match == override_t::MATCH_EXACT;
std::string filter_by_clause = override->filter_by;
@@@ -2821,12 -2756,26 +2806,25 @@@ Option<bool> Index::search(std::vector<
auto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&
field_query_tokens[0].q_include_tokens[0].value == "*";
-
-
+
+ bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);
+
+ // handle phrase searches
+ if (!field_query_tokens[0].q_phrases.empty()) {
+ do_phrase_search(num_search_fields, the_fields, field_query_tokens,
+ sort_fields_std, searched_queries, group_limit, group_by_fields,
+ topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,
+ all_result_ids, all_result_ids_len, groups_processed, curated_ids,
+ excluded_result_ids, excluded_result_ids_size, curated_topster,
+ included_ids_map, is_wildcard_query,
+ filter_result.docs, filter_result.count);
+ if (filter_result.count == 0) {
+ goto process_search_results;
+ }
+ }
-
++
// for phrase query, parser will set field_query_tokens to "*", need to handle that
- if (is_wildcard_query) {
- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
+ if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {
- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
- bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);
if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&
sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&
@@@ -3351,7 -3387,7 +3450,6 @@@ void Index::process_curated_ids(const s
std::sort(all_positions.begin(), all_positions.end());
all_positions.erase(unique(all_positions.begin(), all_positions.end()), all_positions.end());
-- size_t pos_count = 0;
std::map<size_t, std::map<size_t, uint32_t>> new_included_ids_map;
auto included_id_it = included_ids_map.begin();
auto all_pos_it = all_positions.begin();
@@@ -3703,10 -3740,10 +3802,6 @@@ void Index::find_across_fields(const to
result_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, filter_ids, filter_ids_length);
-- const bool prefix_search = previous_token.is_prefix_searched;
-- const uint32_t token_num_typos = previous_token.num_typos;
-- const bool token_prefix = previous_token.is_prefix_searched;
--
auto& token_str = previous_token_str;
auto token_c_str = (const unsigned char*) token_str.c_str();
const size_t token_len = token_str.size() + 1;
@@@ -3802,7 -3883,7 +3941,6 @@@ void Index::search_across_fields(const
// for each token, find the posting lists across all query_by fields
for(size_t ti = 0; ti < query_tokens.size(); ti++) {
-- const bool prefix_search = query_tokens[ti].is_prefix_searched;
const uint32_t token_num_typos = query_tokens[ti].num_typos;
const bool token_prefix = query_tokens[ti].is_prefix_searched;
@@@ -4715,7 -4881,7 +4941,6 @@@ void Index::search_wildcard(filter_node
std::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,
const std::vector<size_t>& geopoint_indices) const {
-- uint32_t token_bits = 0;
const bool check_for_circuit_break = (filter_ids_length > 1000000);
//auto beginF = std::chrono::high_resolution_clock::now();
@@@ -5398,10 -5572,7 +5630,7 @@@ uint64_t Index::get_distinct_id(const s
&& (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {
continue;
}
- // if(!facet_index_v4->contains(field)) {
- // continue;
- // }
--
++
if(search_schema.at(field).is_array()) {
const auto& field_facet_mapping = field_facet_mapping_it->second;
const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);