Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.25-join' of https://github.com/typesense/typesense into facet_index_refactor
krunal1313c6a2c7946bad
diff --cc include/field.hindex 9e8b32a2,4fa6cc6e..696c0bb0--- a/include/field.h+++ b/include/field.h@@@ -715,9 -833,9 +832,11 @@@ struct facetbool sampled = false;+ bool is_wildcard_match = false;+- bool get_range(int64_t key, std::pair<int64_t, std::string>& range_pair)+ bool is_intersected = false;++ bool get_range(std::string key, std::pair<std::string, std::string>& range_pair){if(facet_range_map.empty()){@@@ -765,12 -882,11 +883,10 @@@ struct facet_value_tstruct facet_hash_values_t {uint32_t length = 0;- //uint32_t* hashes = nullptr;- uint64_t* hashes = nullptr;+ std::vector<uint32_t> hashes;facet_hash_values_t() {length = 0;- //hashes = nullptr;- hashes = nullptr;}facet_hash_values_t(facet_hash_values_t&& hash_values) noexcept {@@@ -778,20 -894,17 +894,17 @@@hashes = hash_values.hashes;hash_values.length = 0;- //hash_values.hashes = nullptr;- hash_values.hashes = nullptr;+ hash_values.hashes.clear();}facet_hash_values_t& operator=(facet_hash_values_t&& other) noexcept {if (this != &other) {- //delete[] hashes;- delete[] hashes;+ hashes.clear();hashes = other.hashes;length = other.length;- //other.hashes = nullptr;- other.hashes = nullptr;+ other.hashes.clear();other.length = 0;}@@@ -799,9 -912,8 +912,7 @@@}~facet_hash_values_t() {- //delete [] hashes;- //hashes = nullptr;- delete [] hashes;- hashes = nullptr;+ hashes.clear();}uint64_t size() const {@@@ -809,7 -921,6 +920,6 @@@}uint64_t back() const {- //return hashes[length - 1];- return hashes[length - 1];+ return hashes.back();}-};+};diff --cc include/index.hindex 3cc57b42,332e9fa2..b2b1f8c4--- a/include/index.h+++ b/include/index.h@@@ -187,24 -187,9 +188,9 @@@ struct search_args};};- enum index_operation_t {- CREATE,- UPSERT,- UPDATE,- EMPLACE,- DELETE- };-- enum class DIRTY_VALUES {- REJECT = 1,- DROP = 2,- COERCE_OR_REJECT = 3,- COERCE_OR_DROP = 4,- };-struct offsets_facet_hashes_t {std::unordered_map<std::string, std::vector<uint32_t>> offsets;- std::vector<uint64_t> facet_hashes;+ //std::vector<uint32_t> facet_hashes;};struct index_record {@@@ -514,23 -506,23 +510,19 @@@ privatestatic void tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,- std::vector<uint32_t>& facet_hashes*/);- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes);++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets);static void tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,- std::vector<uint32_t>& facet_hashes*/);- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes);++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets);void collate_included_ids(const std::vector<token_t>& q_included_tokens,const std::map<size_t, std::map<size_t, uint32_t>> & included_ids_map,Topster* curated_topster, std::vector<std::vector<art_leaf*>> & searched_queries) const;-- static uint64_t facet_token_hash(const field & a_field, const std::string &token);--- static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);+ static void compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type);static void get_doc_changes(const index_operation_t op, nlohmann::json &update_doc,const nlohmann::json &old_doc, nlohmann::json &new_doc, nlohmann::json &del_doc);diff --cc src/collection.cppindex dd8ccbc7,90ca8b1a..19aa663a--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -1842,129 -1932,104 +1926,129 @@@ Option<nlohmann::json> Collection::searLOG (ERROR) << "range_id not found in result map.";}}- }-- auto the_field = search_schema.at(a_facet.field_name);- // keep only top K facets- auto max_facets = std::min(max_facet_values, facet_hash_counts.size());- std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,- facet_hash_counts.end(), Collection::facet_count_compare);- for(size_t fi = 0; fi < max_facets; fi++) {-- if(a_facet.is_range_query){- break;+ } else if(a_facet.is_intersected) {- LOG(INFO) << "used intersection";++ //LOG(INFO) << "used intersection";+ std::vector<std::pair<std::string, uint32_t>> facet_counts;++ for (const auto & kv : a_facet.result_map) {+ facet_counts.emplace_back(std::make_pair(kv.first, kv.second.count));}++ auto max_facets = std::min(max_facet_values, facet_counts.size());+ std::sort(facet_counts.begin(), facet_counts.end(),+ [&](const auto& p1, const auto& p2) {+ return p1.second > p2.second;+ });- // remap facet value hash with actual string- auto & kv = facet_hash_counts[fi];- auto & facet_count = kv.second;- // fetch actual facet value from representative doc id- const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);- nlohmann::json document;- const Option<bool> & document_op = get_document_from_store(seq_id_key, document);- if(!document_op.ok()) {- LOG(ERROR) << "Facet fetch error. " << document_op.error();- continue;+ for(int i = 0; i < max_facets; ++i) {+ const auto& kv = facet_counts[i];+ facet_value_t facet_value = { kv.first, kv.first, kv.second};+ facet_values.emplace_back(facet_value);}- std::string value;- bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);- if(!facet_found) {- continue;+ } else {- LOG(INFO) << "used hashes";++ //LOG(INFO) << "used hashes";+ std::vector<std::pair<uint32_t, facet_count_t>> facet_hash_counts;++ for (const auto & kv : a_facet.result_map) {+ facet_hash_counts.emplace_back(std::make_pair(std::stoul(kv.first), kv.second));}- std::unordered_map<std::string, size_t> ftoken_pos;- std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];- for(size_t ti = 0; ti < ftokens.size(); ti++) {- if(the_field.is_bool()) {- if(ftokens[ti] == "1") {- ftokens[ti] = "true";- } else {- ftokens[ti] = "false";- }++ auto the_field = search_schema.at(a_facet.field_name);+ // keep only top K facets+ auto max_facets = std::min(max_facet_values, facet_hash_counts.size());++ std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,+ facet_hash_counts.end(), Collection::facet_count_compare);++ for(size_t fi = 0; fi < max_facets; fi++) {+ // remap facet value hash with actual string+ auto & kv = facet_hash_counts[fi];+ auto & facet_count = kv.second;+ // fetch actual facet value from representative doc id+ const std::string& seq_id_key = get_seq_id_key((uint32_t) facet_count.doc_id);+ nlohmann::json document;+ const Option<bool> & document_op = get_document_from_store(seq_id_key, document);+ if(!document_op.ok()) {+ LOG(ERROR) << "Facet fetch error. " << document_op.error();+ continue;}- const std::string& resolved_token = ftokens[ti];- ftoken_pos[resolved_token] = ti;- }- const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();- // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text- bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);- bool normalise = is_cyrillic ? false : true;- Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);- // secondary tokenizer used for specific languages that requires transliteration- // we use 2 tokenizers so that the original text offsets are available for highlighting- Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);- std::string raw_token;- size_t raw_token_index = 0, tok_start = 0, tok_end = 0;- // need an ordered map here to ensure that it is ordered by the key (start offset)- std::map<size_t, size_t> token_offsets;- size_t prefix_token_start_index = 0;- while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {- if(is_cyrillic) {- word_tokenizer.tokenize(raw_token);+ std::string value;+ bool facet_found = facet_value_to_string(a_facet, facet_count, document, value);+ if(!facet_found) {+ continue;}- auto token_pos_it = ftoken_pos.find(raw_token);- if(token_pos_it != ftoken_pos.end()) {- token_offsets[tok_start] = tok_end;- if(raw_token == last_full_q_token) {- prefix_token_start_index = tok_start;+ std::unordered_map<std::string, size_t> ftoken_pos;+ std::vector<string>& ftokens = a_facet.hash_tokens[kv.first];+ //LOG(INFO) << "working on hash_tokens for hash " << kv.first << " with size " << ftokens.size();+ for(size_t ti = 0; ti < ftokens.size(); ti++) {+ if(the_field.is_bool()) {+ if(ftokens[ti] == "1") {+ ftokens[ti] = "true";+ } else {+ ftokens[ti] = "false";+ }}+ const std::string& resolved_token = ftokens[ti];+ ftoken_pos[resolved_token] = ti;}- }- auto offset_it = token_offsets.begin();- size_t i = 0;- std::stringstream highlightedss;- // loop until end index, accumulate token and complete highlighting- while(i < value.size()) {- if(offset_it != token_offsets.end()) {- if (i == offset_it->first) {- highlightedss << highlight_start_tag;- // do prefix highlighting for non-dropped last token- size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?- facet_query_last_token.size() :- (offset_it->second - i + 1);- if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {- token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());- } else {- token_len = (offset_it->second - i + 1);+ const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();+ // 2 passes: first identify tokens that need to be highlighted and then construct highlighted text+ bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);+ bool normalise = is_cyrillic ? false : true;+ Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);+ // secondary tokenizer used for specific languages that requires transliteration+ // we use 2 tokenizers so that the original text offsets are available for highlighting+ Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);+ std::string raw_token;+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;+ // need an ordered map here to ensure that it is ordered by the key (start offset)+ std::map<size_t, size_t> token_offsets;+ size_t prefix_token_start_index = 0;+ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ if(is_cyrillic) {+ word_tokenizer.tokenize(raw_token);+ }+ auto token_pos_it = ftoken_pos.find(raw_token);+ if(token_pos_it != ftoken_pos.end()) {+ token_offsets[tok_start] = tok_end;+ if(raw_token == last_full_q_token) {+ prefix_token_start_index = tok_start;}- for(size_t j = 0; j < token_len; j++) {- highlightedss << value[i + j];+ }+ }+ //LOG(INFO) << "token_offsets size " << token_offsets.size();+ auto offset_it = token_offsets.begin();+ size_t i = 0;+ std::stringstream highlightedss;+ // loop until end index, accumulate token and complete highlighting+ while(i < value.size()) {+ if(offset_it != token_offsets.end()) {+ if (i == offset_it->first) {+ highlightedss << highlight_start_tag;+ // do prefix highlighting for non-dropped last token+ size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?+ facet_query_last_token.size() :+ (offset_it->second - i + 1);+ if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {+ token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());+ } else {+ token_len = (offset_it->second - i + 1);+ }+ for(size_t j = 0; j < token_len; j++) {+ highlightedss << value[i + j];+ }+ highlightedss << highlight_end_tag;+ offset_it++;+ i += token_len;+ continue;}- highlightedss << highlight_end_tag;- offset_it++;- i += token_len;- continue;}+ highlightedss << value[i];+ i++;}- highlightedss << value[i];- i++;+ facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};+ facet_values.emplace_back(facet_value);}- facet_value_t facet_value = {value, highlightedss.str(), facet_count.count};- facet_values.emplace_back(facet_value);}std::stable_sort(facet_values.begin(), facet_values.end(), Collection::facet_count_str_compare);diff --cc src/facet_index.cppindex 7875f091,00000000..3a8902damode 100644,000000..100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@@ -1,159 -1,0 +1,160 @@@+#include "facet_index.h"+#include <tokenizer.h>+#include "string_utils.h"++uint32_t facet_index_t::insert(const std::string& field, const std::string& value, uint32_t id) {+ auto& facet_index_map = facet_field_map[field].facet_index_map;+ uint32_t index;+ const auto sv = value.substr(0, 100);- if(facet_index_map.count(sv) == 0) {++ const auto it = facet_index_map.find(sv);++ if(it == facet_index_map.end()) {+ index = ++count_index;++ facet_index_struct fis {};+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));+ fis.index = index;+ facet_index_map.emplace(sv, fis);+ }else {+ auto ids = facet_index_map[sv].id_list_ptr;+ if (!ids_t::contains(ids, id)) {+ ids_t::upsert(ids, id);+ facet_index_map[sv].id_list_ptr = ids;+ }+ index = facet_index_map[sv].index;+ }+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;+ auto& counter_list = facet_field_map[field].counter_list;+ count_list node(sv, facet_count);++ if(counter_list.empty()) {+ counter_list.emplace_back(count_list(sv, facet_count));+ } else {+ auto it = counter_list.begin();+ //remove node from list+ for(it = counter_list.begin(); it != counter_list.end(); ++it) {+ if(it->facet_value == sv) {+ //found facet in first node+ counter_list.erase(it);+ break;+ }+ }++ //find position in list and add node with updated count+ count_list node(sv, facet_count);++ for(it = counter_list.begin(); it != counter_list.end(); ++it) {+ // LOG (INFO) << "inserting in middle or front facet " << node.facet_value+ // << " with count " << node.count;+ if(it->count <= facet_count) {+ counter_list.emplace(it, node);+ break;+ }+ }+ if(it == counter_list.end()) {+ // LOG (INFO) << "inserting at last facet " << node.facet_value+ // << " with count " << node.count;+ counter_list.emplace_back(node);+ }+ }++ return index;+}++bool facet_index_t::contains(const std::string& field) {++ const auto& facet_field_it = facet_field_map.find(field);+ if(facet_field_it == facet_field_map.end()) {+ return false;+ }++ return true;+}++void facet_index_t::erase(const std::string& field) {+ const auto it = facet_field_map.find(field);+ if(it != facet_field_map.end()) {+ facet_field_map.erase(field);+ }+}++size_t facet_index_t::get_facet_count(const std::string& field) {+ const auto it = facet_field_map.find(field);++ if(it == facet_field_map.end()) {+ return 0;+ }++ return it->second.counter_list.size();+}++//returns the count of matching seq_ids from result array+int facet_index_t::intersect(const std::string& field, const uint32_t* result_ids,+ int result_ids_len, int max_facet_count,+ std::map<std::string, uint32_t>& found, bool is_wildcard_no_filter_query) {+ //LOG (INFO) << "intersecting field " << field;++ const auto& facet_field_it = facet_field_map.find(field);+ if(facet_field_it == facet_field_map.end()) {+ return 0;+ }++ const auto facet_index_map = facet_field_it->second.facet_index_map;+ const auto counter_list = facet_field_it->second.counter_list;++ // LOG (INFO) << "facet_index_map size " << facet_index_map.size()+ // << " , counter_list size " << counter_list.size();++ std::vector<uint32_t> id_list;+ for(const auto& counter_list_it : counter_list) {+ // LOG (INFO) << "checking ids in facet_value " << counter_list_it.facet_value+ // << " having total count " << counter_list_it.count;+ int count = 0;++ if(is_wildcard_no_filter_query) {+ count = counter_list_it.count;+ } else {+ auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;+ ids_t::uncompress(ids, id_list);+ const auto ids_len = id_list.size();+ for(int i = 0; i < result_ids_len; ++i) {+ if(std::binary_search(id_list.begin(), id_list.end(), result_ids[i])) {+ ++count;+ }+ }+ id_list.clear();+ }++ if(count) {+ found[counter_list_it.facet_value] = count;+ if(found.size() == max_facet_count) {+ break;+ }+ }+ }++ return found.size();+}++std::string facet_index_t::get_facet_by_count_index(const std::string& field, uint32_t count_index) {++ const auto& facet_field_it = facet_field_map.find(field);++ if(facet_field_it == facet_field_map.end()) {+ return "";+ }+ std::string result = "";+ auto facet_index_map = facet_field_it->second.facet_index_map;++ for(auto it = facet_index_map.begin(); it != facet_index_map.end(); ++it) {+ if(it.value().index == count_index) {+ result = it.key();+ }+ }+ return result;+}++facet_index_t::~facet_index_t() {+ facet_field_map.clear();+}+diff --cc src/index.cppindex 29f8e4fd,a24e6cb7..cd7e7abd--- a/src/index.cpp+++ b/src/index.cpp@@@ -814,97 -651,30 +656,87 @@@ void Index::index_field_in_memory(consif(afield.facet) {if(afield.is_array()) {- facet_hash_values_t fhashvalues;- fhashvalues.length = field_index_it->second.facet_hashes.size();- fhashvalues.hashes = new uint64_t[field_index_it->second.facet_hashes.size()];-- for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {- fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];+ facet_hash_values_t fhashvalues;++ if(afield.type == field_types::INT32_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ int32_t raw_val = document[afield.name][i].get<int32_t>();+ value = std::to_string(raw_val);+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }}-- auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ else if(afield.type == field_types::INT64_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ int64_t raw_val = document[afield.name][i].get<int64_t>();+ value = std::to_string(raw_val);+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ else if(afield.type == field_types::STRING_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ value = document[afield.name][i];+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ else if(afield.type == field_types::FLOAT_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ float raw_val = document[afield.name][i].get<float>();+ value = StringUtils::float_to_str(raw_val);+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }+ }+ else if(afield.type == field_types::BOOL_ARRAY) {+ for(int i = 0; i < document[afield.name].size(); ++i) {+ value = std::to_string(document[afield.name][i].get<bool>());- //value = (value == "1") ? "true" : "false";+ auto index = facet_index_v4->insert(afield.name, value, seq_id);+ fhashvalues.hashes.emplace_back(index);+ }}+ fhashvalues.length = fhashvalues.hashes.size();- //LOG(INFO) << "fhashvalues.length " << fhashvalues.length;- //fhashvalues.length = field_index_it->second.facet_hashes.size();- //fhashvalues.hashes = new uint32_t[field_index_it->second.facet_hashes.size()];-- // for(size_t i = 0; i < field_index_it->second.facet_hashes.size(); i++) {- // fhashvalues.hashes[i] = field_index_it->second.facet_hashes[i];- // }++ auto& facet_dim_index = facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ } else {+ facet_dim_index->emplace(seq_id, std::move(fhashvalues));+ }} else {- uint64_t fhash;- fhash = field_index_it->second.facet_hashes[0];-- auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];- if(facet_dim_index == nullptr) {- LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;- } else {- facet_dim_index->emplace(seq_id, std::move(fhash));+ uint32_t fhash;++ if(afield.type == field_types::INT32) {+ int32_t raw_val = document[afield.name].get<int32_t>();+ value = std::to_string(raw_val);+ fhash = facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::INT64) {+ int64_t raw_val = document[afield.name].get<int64_t>();+ value = std::to_string(raw_val);+ fhash = facet_index_v4->insert(afield.name, value, seq_id);}+ else if(afield.type == field_types::STRING) {+ value = document[afield.name];+ fhash = facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::FLOAT) {+ float raw_val = document[afield.name].get<float>();+ value = StringUtils::float_to_str(raw_val);+ fhash = facet_index_v4->insert(afield.name, value, seq_id);+ }+ else if(afield.type == field_types::BOOL) {+ value = std::to_string(document[afield.name].get<bool>());- //value = (value == "1") ? "true" : "false";+ fhash = facet_index_v4->insert(afield.name, value, seq_id);+ }- //fhash = field_index_it->second.facet_hashes[0];++ auto& facet_dim_index = single_val_facet_index_v3[afield.name][seq_id % ARRAY_FACET_DIM];+ if(facet_dim_index == nullptr) {+ LOG(ERROR) << "Error, facet index not initialized for field " << afield.name;+ } else {+ facet_dim_index->emplace(seq_id, std::move(fhash));+ }}}@@@ -1185,68 -966,73 +1028,40 @@@}}--uint64_t Index::facet_token_hash(const field & a_field, const std::string &token) {-- // for integer/float use their native values-- uint64_t hash = 0;---- if(a_field.is_float()) {-- float f = std::stof(token);-- reinterpret_cast<float&>(hash) = f; // store as int without loss of precision-- } else if(a_field.is_integer() || a_field.is_bool()) {-- hash = atoll(token.c_str());-- } else {-- // string field-- hash = StringUtils::hash_wy(token.c_str(), token.size());-- }---- return hash;--}--void Index::tokenize_string_with_facets(const std::string& text, bool is_facet, const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,- std::vector<uint32_t>& facet_hashes*/) {- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes) {++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets) {Tokenizer tokenizer(text, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);std::string token;std::string last_token;size_t token_index = 0;-- uint64_t facet_hash = 1;while(tokenizer.next(token, token_index)) {if(token.empty()) {continue;}+ if(token.size() > 100) {+ token.erase(100);+ }+token_to_offsets[token].push_back(token_index + 1);last_token = token;- // if(is_facet) {- // uint64_t token_hash = Index::facet_token_hash(a_field, token);- // if(token_index == 0) {- // facet_hash = token_hash;- // } else {- // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);- // }- // }-- if(is_facet) {- uint64_t token_hash = Index::facet_token_hash(a_field, token);- if(token_index == 0) {- facet_hash = token_hash;- } else {- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);- }- }}if(!token_to_offsets.empty()) {// push 0 for the last occurring token (used for exact match ranking)token_to_offsets[last_token].push_back(0);}--- // if(is_facet) {- // facet_hashes.push_back(facet_hash);- // }- if(is_facet) {- facet_hashes.push_back(facet_hash);- }}void Index::tokenize_string_array_with_facets(const std::vector<std::string>& strings, bool is_facet,const field& a_field,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators,- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets/*,- std::vector<uint32_t>& facet_hashes*/) {- std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets,- std::vector<uint64_t>& facet_hashes) {++ std::unordered_map<std::string, std::vector<uint32_t>>& token_to_offsets) {for(size_t array_index = 0; array_index < strings.size(); array_index++) {const std::string& str = strings[array_index];@@@ -1255,7 -1041,7 +1070,6 @@@Tokenizer tokenizer(str, true, !a_field.is_string(), a_field.locale, symbols_to_index, token_separators);std::string token, last_token;size_t token_index = 0;-- uint64_t facet_hash = 1;// iterate and append offset positionswhile(tokenizer.next(token, token_index)) {@@@ -1266,21 -1056,21 +1084,8 @@@token_to_offsets[token].push_back(token_index + 1);token_set.insert(token);last_token = token;--- // if(is_facet) {- // uint64_t token_hash = Index::facet_token_hash(a_field, token);- // if(token_index == 0) {- // facet_hash = token_hash;- // } else {- // facet_hash = StringUtils::hash_combine(facet_hash, token_hash);- // }- // }- if(is_facet) {- uint64_t token_hash = Index::facet_token_hash(a_field, token);- if(token_index == 0) {- facet_hash = token_hash;- } else {- facet_hash = StringUtils::hash_combine(facet_hash, token_hash);- }- }- }-- if(is_facet) {- facet_hashes.push_back(facet_hash);}- // if(is_facet) {- // facet_hashes.push_back(facet_hash);- // }-if(token_set.empty()) {continue;}@@@ -1363,120 -1153,91 +1168,129 @@@ void Index::do_facets(std::vector<facetsize_t mod_value = 100 / facet_sample_percent;- facet_map_t::iterator facet_map_it;- single_val_facet_map_t::iterator single_facet_map_it;- uint64_t fhash = 0;- size_t facet_hash_count = 1;- const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);- const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);+ const auto facet_records = facet_index_v4->get_facet_count(a_facet.field_name);- if((field_facet_mapping_it == facet_index_v3.end())- && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {- continue;- }+ if(results_size && facet_records && (facet_records <= 10 || is_wildcard_query == true)+ && use_facet_query == false && group_limit == 0 && no_filters_provided == true) {+ //LOG(INFO) << "Using intersection to find facets";+ a_facet.is_intersected = true;- for(size_t i = 0; i < results_size; i++) {- // if sampling is enabled, we will skip a portion of the results to speed up things- if(estimate_facets) {- if(i % mod_value != 0) {- continue;- }- }-- uint32_t doc_seq_id = result_ids[i];- if(facet_field.is_array()) {- const auto& field_facet_mapping = field_facet_mapping_it->second;- facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {- continue;+ std::map<std::string, uint32_t> facet_results;+ facet_index_v4->intersect(a_facet.field_name, result_ids,+ results_size, max_facet_count, facet_results, is_wildcard_query & no_filters_provided);++ for(const auto& kv : facet_results) {+ //range facet processing+ if(a_facet.is_range_query) {+ const auto doc_val = kv.first;+ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count = kv.second;+ }+ } else {+ facet_count_t& facet_count = a_facet.result_map[kv.first];+ facet_count.count = kv.second;}- facet_hash_count = facet_map_it->second.size();- } else {- const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;- single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);-- if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {++ if(should_compute_stats) {+ //LOG(INFO) << "Computing facet stas for facet " << a_facet.field_name;+ for(int i = 0; i < kv.second; ++i) {+ compute_facet_stats(a_facet, kv.first, facet_field.type);+ }+ }+ }+ } else {+ facet_map_t::iterator facet_map_it;+ single_val_facet_map_t::iterator single_facet_map_it;+ uint32_t fhash = 0;+ size_t facet_hash_count = 1;++ const auto& field_facet_mapping_it = facet_index_v3.find(a_facet.field_name);+ const auto& field_single_val_facet_mapping_it = single_val_facet_index_v3.find(a_facet.field_name);+++ if((field_facet_mapping_it == facet_index_v3.end())++ && (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {+ continue;- }- facet_hash_count = 1;- fhash = single_facet_map_it->second;+ }-- const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+- if(((i + 1) % 16384) == 0) {- RETURN_CIRCUIT_BREAKER- }+ for(size_t i = 0; i < results_size; i++) {+ // if sampling is enabled, we will skip a portion of the results to speed up things+ if(estimate_facets) {+ if(i % mod_value != 0) {+ continue;+ }+ }- for(size_t j = 0; j < facet_hash_count; j++) {-+ uint32_t doc_seq_id = result_ids[i];if(facet_field.is_array()) {- fhash = facet_map_it->second.hashes[j];+ const auto& field_facet_mapping = field_facet_mapping_it->second;+ facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);++ if(facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }+ facet_hash_count = facet_map_it->second.size();+ } else {+ const auto& field_facet_mapping = field_single_val_facet_mapping_it->second;+ single_facet_map_it = field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->find(doc_seq_id);++ if(single_facet_map_it == field_facet_mapping[doc_seq_id % ARRAY_FACET_DIM]->end()) {+ continue;+ }+ facet_hash_count = 1;+ fhash = single_facet_map_it->second;}-- if(should_compute_stats) {- compute_facet_stats(a_facet, fhash, facet_field.type);++ const uint64_t distinct_id = group_limit ? get_distinct_id(group_by_fields, doc_seq_id) : 0;+ //LOG(INFO) << "facet_hash_count " << facet_hash_count;++ if(((i + 1) % 16384) == 0) {++ RETURN_CIRCUIT_BREAKER+ }- if(a_facet.is_range_query) {- auto sort_index_it = sort_index.find(a_facet.field_name);-- if(sort_index_it != sort_index.end()){- auto doc_id_val_map = sort_index_it->second;- auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);-- if(doc_seq_id_it != doc_id_val_map->end()){-- int64_t doc_val = doc_seq_id_it->second;- std::pair<int64_t, std::string> range_pair {};- if(a_facet.get_range(doc_val, range_pair)) {- int64_t range_id = range_pair.first;- facet_count_t& facet_count = a_facet.result_map[range_id];- facet_count.count += 1;- }- }+++ for(size_t j = 0; j < facet_hash_count; j++) {+ if(facet_field.is_array()) {+ fhash = facet_map_it->second.hashes[j];}- } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {- facet_count_t& facet_count = a_facet.result_map[fhash];- //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;- facet_count.doc_id = doc_seq_id;- facet_count.array_pos = j;- if(group_limit) {- a_facet.hash_groups[fhash].emplace(distinct_id);- } else {- facet_count.count += 1;+ if(should_compute_stats) {+ std::string fvalue =+ facet_index_v4->get_facet_by_count_index(a_facet.field_name, fhash);+ if(!fvalue.empty()) {+ compute_facet_stats(a_facet, fvalue, facet_field.type);+ }}- if(use_facet_query) {- a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);+ if(a_facet.is_range_query) {+ auto sort_index_it = sort_index.find(a_facet.field_name);+ if(sort_index_it != sort_index.end()){+ auto doc_id_val_map = sort_index_it->second;+ auto doc_seq_id_it = doc_id_val_map->find(doc_seq_id);+ if(doc_seq_id_it != doc_id_val_map->end()){++ std::string doc_val = std::to_string(doc_seq_id_it->second);+ std::pair<std::string, std::string> range_pair {};+ if(a_facet.get_range(doc_val, range_pair)) {+ const auto& range_id = range_pair.first;+ facet_count_t& facet_count = a_facet.result_map[range_id];+ facet_count.count += 1;+ }+ }+ }+ } else if(!use_facet_query || fquery_hashes.find(fhash) != fquery_hashes.end()) {+ std::string fhash_str = std::to_string(fhash);+ facet_count_t& facet_count = a_facet.result_map[fhash_str];+ //LOG(INFO) << "field: " << a_facet.field_name << ", doc id: " << doc_seq_id << ", hash: " << fhash;+ facet_count.doc_id = doc_seq_id;+ facet_count.array_pos = j;+ if(group_limit) {+ a_facet.hash_groups[fhash].emplace(distinct_id);+ } else {+ facet_count.count += 1;+ }+ if(use_facet_query) {+ //LOG (INFO) << "adding hash tokens for hash " << fhash;+ a_facet.hash_tokens[fhash] = fquery_hashes.at(fhash);+ }}}}@@@ -1659,7 -1421,7 +1474,6 @@@ void Index::search_candidates(const uin);// We fetch offset positions only for multi token query-- bool fetch_offsets = (query_suggestion.size() > 1);bool single_exact_query_token = false;if(total_cost == 0 && query_suggestion.size() == query_tokens.size() == 1) {@@@ -1856,7 -1706,7 +1758,6 @@@ Option<bool> Index::do_filtering(filterif (is_polygon) {const int num_verts = int(filter_value_parts.size()) / 2;std::vector<S2Point> vertices;-- double sum = 0.0;for (size_t point_index = 0; point_index < size_t(num_verts);point_index++) {@@@ -2515,11 -2453,11 +2505,8 @@@ void Index::process_filter_overrides(co// we will cover both original query and synonymsstd::vector<std::string> rule_parts;- StringUtils::split(override->rule.query, rule_parts, " ");-- uint32_t* field_override_ids = nullptr;- size_t field_override_ids_len = 0;+ StringUtils::split(override->rule.normalized_query, rule_parts, " ");- uint32_t* field_override_ids = nullptr;- size_t field_override_ids_len = 0;-bool exact_rule_match = override->rule.match == override_t::MATCH_EXACT;std::string filter_by_clause = override->filter_by;@@@ -2821,12 -2756,26 +2806,25 @@@ Option<bool> Index::search(std::vector<auto is_wildcard_query = !field_query_tokens.empty() && !field_query_tokens[0].q_include_tokens.empty() &&field_query_tokens[0].q_include_tokens[0].value == "*";--++ bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);++ // handle phrase searches+ if (!field_query_tokens[0].q_phrases.empty()) {+ do_phrase_search(num_search_fields, the_fields, field_query_tokens,+ sort_fields_std, searched_queries, group_limit, group_by_fields,+ topster, sort_order, field_values, geopoint_indices, curated_ids_sorted,+ all_result_ids, all_result_ids_len, groups_processed, curated_ids,+ excluded_result_ids, excluded_result_ids_size, curated_topster,+ included_ids_map, is_wildcard_query,+ filter_result.docs, filter_result.count);+ if (filter_result.count == 0) {+ goto process_search_results;+ }+ }-++// for phrase query, parser will set field_query_tokens to "*", need to handle that- if (is_wildcard_query) {- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);+ if (is_wildcard_query && field_query_tokens[0].q_phrases.empty()) {- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);- bool no_filters_provided = (filter_tree_root == nullptr && filter_result.count == 0);if(no_filters_provided && facets.empty() && curated_ids.empty() && vector_query.field_name.empty() &&sort_fields_std.size() == 1 && sort_fields_std[0].name == sort_field_const::seq_id &&@@@ -3351,7 -3387,7 +3450,6 @@@ void Index::process_curated_ids(const sstd::sort(all_positions.begin(), all_positions.end());all_positions.erase(unique(all_positions.begin(), all_positions.end()), all_positions.end());-- size_t pos_count = 0;std::map<size_t, std::map<size_t, uint32_t>> new_included_ids_map;auto included_id_it = included_ids_map.begin();auto all_pos_it = all_positions.begin();@@@ -3703,10 -3740,10 +3802,6 @@@ void Index::find_across_fields(const toresult_iter_state_t istate(exclude_token_ids, exclude_token_ids_size, filter_ids, filter_ids_length);-- const bool prefix_search = previous_token.is_prefix_searched;-- const uint32_t token_num_typos = previous_token.num_typos;-- const bool token_prefix = previous_token.is_prefix_searched;--auto& token_str = previous_token_str;auto token_c_str = (const unsigned char*) token_str.c_str();const size_t token_len = token_str.size() + 1;@@@ -3802,7 -3883,7 +3941,6 @@@ void Index::search_across_fields(const// for each token, find the posting lists across all query_by fieldsfor(size_t ti = 0; ti < query_tokens.size(); ti++) {-- const bool prefix_search = query_tokens[ti].is_prefix_searched;const uint32_t token_num_typos = query_tokens[ti].num_typos;const bool token_prefix = query_tokens[ti].is_prefix_searched;@@@ -4715,7 -4881,7 +4941,6 @@@ void Index::search_wildcard(filter_nodestd::array<spp::sparse_hash_map<uint32_t, int64_t>*, 3>& field_values,const std::vector<size_t>& geopoint_indices) const {-- uint32_t token_bits = 0;const bool check_for_circuit_break = (filter_ids_length > 1000000);//auto beginF = std::chrono::high_resolution_clock::now();@@@ -5398,10 -5572,7 +5630,7 @@@ uint64_t Index::get_distinct_id(const s&& (field_single_val_facet_mapping_it == single_val_facet_index_v3.end())) {continue;}- // if(!facet_index_v4->contains(field)) {- // continue;- // }--++if(search_schema.at(field).is_array()) {const auto& field_facet_mapping = field_facet_mapping_it->second;const auto& facet_hashes_it = field_facet_mapping[seq_id % ARRAY_FACET_DIM]->find(seq_id);