Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Skeleton for nested field indexing and highlighting.
Kishore Nallanbef79000e197
diff --git a/include/collection.h b/include/collection.hindex 30eed5f6..a547e2e8 100644--- a/include/collection.h+++ b/include/collection.h@@ -65,6 +65,24 @@ private:}};+ struct match_index_t {+ Match match;+ uint64_t match_score = 0;+ size_t index;++ match_index_t(Match match, uint64_t match_score, size_t index): match(match), match_score(match_score),+ index(index) {++ }++ bool operator<(const match_index_t& a) const {+ if(match_score != a.match_score) {+ return match_score > a.match_score;+ }+ return index < a.index;+ }+ };+const std::string name;const std::atomic<uint32_t> collection_id;@@ -80,7 +98,7 @@ private:std::vector<field> fields;- std::unordered_map<std::string, field> search_schema;+ tsl::htrie_map<char, field> search_schema;std::map<std::string, override_t> overrides;@@ -112,6 +130,7 @@ private:const tsl::htrie_map<char, token_leaf>& qtoken_leaves,const std::vector<std::string>& q_tokens,const KV* field_order_kv, const nlohmann::json &document,+ nlohmann::json& highlight_doc,StringUtils & string_utils,const size_t snippet_threshold,const size_t highlight_affix_num_tokens,@@ -134,7 +153,7 @@ private:static Option<bool> detect_new_fields(nlohmann::json& document,const DIRTY_VALUES& dirty_values,- const std::unordered_map<std::string, field>& schema,+ const tsl::htrie_map<char, field>& schema,const std::unordered_map<std::string, field>& dyn_fields,const std::string& fallback_field_type,std::vector<field>& new_fields);@@ -167,15 +186,15 @@ private:Option<bool> persist_collection_meta();- Option<bool> batch_alter_data(const std::unordered_map<std::string, field>& schema_additions,+ Option<bool> batch_alter_data(const tsl::htrie_map<char, field>& schema_additions,const std::unordered_map<std::string, field>& new_dynamic_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type,const bool do_validation);Option<bool> validate_alter_payload(nlohmann::json& schema_changes,- std::unordered_map<std::string, field>& schema_additions,- std::unordered_map<std::string, field>& schema_reindex,+ tsl::htrie_map<char, field>& schema_additions,+ tsl::htrie_map<char, field>& schema_reindex,std::unordered_map<std::string, field>& addition_dynamic_fields,std::unordered_map<std::string, field>& reindex_dynamic_fields,std::vector<field>& del_fields,@@ -190,6 +209,23 @@ private:void populate_text_match_info(nlohmann::json& info, uint64_t match_score) const;+ static void remove_flat_fields(nlohmann::json& document);++ bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,+ const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,+ highlight_t& highlight, StringUtils & string_utils, bool is_cyrillic,+ const size_t highlight_affix_num_tokens,+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves,+ int last_valid_offset_index, const Match& match,+ const std::string& last_raw_q_token, bool highlight_fully, const size_t snippet_threshold,+ bool is_infix_search, std::vector<std::string>& raw_query_tokens, size_t last_valid_offset,+ const std::string& highlight_start_tag, const std::string& highlight_end_tag,+ const uint8_t* index_symbols, const match_index_t& match_index) const;++ static Option<bool> extract_field_name(const std::string& field_name,+ const tsl::htrie_map<char, field>& search_schema,+ std::vector<std::string>& processed_search_fields);+public:enum {MAX_ARRAY_MATCHES = 5};@@ -254,7 +290,7 @@ public:std::unordered_map<std::string, field> get_dynamic_fields();- std::unordered_map<std::string, field> get_schema();+ tsl::htrie_map<char, field> get_schema();std::string get_default_sorting_field();@@ -272,8 +308,8 @@ public:Option<uint32_t> index_in_memory(nlohmann::json & document, uint32_t seq_id,const index_operation_t op, const DIRTY_VALUES& dirty_values);- static void prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string> & include_fields,- const spp::sparse_hash_set<std::string> & exclude_fields);+ static void prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,+ const tsl::htrie_set<char>& exclude_names, std::string parent_name = "", size_t depth = 0);const Index* _get_index() const;@@ -399,8 +435,8 @@ public:size_t snippet_start_offset) ;void process_highlight_fields(const std::vector<std::string>& search_fields,- const spp::sparse_hash_set<std::string>& exclude_fields,- const spp::sparse_hash_set<std::string>& include_fields,+ const tsl::htrie_set<char>& exclude_fields,+ const tsl::htrie_set<char>& include_fields,const string& highlight_fields,const std::string& highlight_full_fields,const std::vector<enable_t>& infixes,@@ -417,3 +453,33 @@ public:std::vector<std::string>& reordered_search_fields) const;};+template<class T>+bool highlight_nested_field(const nlohmann::json& doc, nlohmann::json& obj,+ std::vector<std::string>& path_parts, size_t path_index, T func) {+ if(path_index == path_parts.size()) {+ // end of path: guaranteed to be a string+ if(!obj.is_string()) {+ return false;+ }++ func(obj);+ }++ const std::string& fragment = path_parts[path_index];+ const auto& it = obj.find(fragment);++ if(it != obj.end()) {+ if(it.value().is_array()) {+ bool resolved = false;+ for(auto& ele: it.value()) {+ resolved |= highlight_nested_field(doc, ele, path_parts, path_index + 1, func);+ }+ return resolved;+ } else {+ return highlight_nested_field(doc, it.value(), path_parts, path_index + 1, func);+ }+ } {+ return false;+ }+}+diff --git a/include/field.h b/include/field.hindex c18179af..7a0a94fa 100644--- a/include/field.h+++ b/include/field.h@@ -7,11 +7,14 @@#include "string_utils.h"#include "logger.h"#include <sparsepp.h>+#include <tsl/htrie_map.h>#include "json.hpp"namespace field_types {// first field value indexed will determine the typestatic const std::string AUTO = "auto";+ static const std::string OBJECT = "object";+ static const std::string OBJECT_ARRAY = "object[]";static const std::string STRING = "string";static const std::string INT32 = "int32";@@ -52,11 +55,15 @@ struct field {bool sort;bool infix;+ bool nested; // field inside an object+ bool nested_array; // field inside an array of objects that is forced to be an array+field() {}field(const std::string &name, const std::string &type, const bool facet, const bool optional = false,bool index = true, std::string locale = "", int sort = -1, int infix = -1) :- name(name), type(type), facet(facet), optional(optional), index(index), locale(locale) {+ name(name), type(type), facet(facet), optional(optional), index(index), locale(locale),+ nested(false), nested_array(false) {if(sort != -1) {this->sort = bool(sort);@@ -67,6 +74,14 @@ struct field {this->infix = (infix != -1) ? bool(infix) : false;}+ bool operator<(const field& f) const {+ return name < f.name;+ }++ bool operator==(const field& f) const {+ return name == f.name;+ }+bool is_auto() const {return (type == field_types::AUTO);}@@ -112,6 +127,10 @@ struct field {return (type == field_types::GEOPOINT || type == field_types::GEOPOINT_ARRAY);}+ bool is_object() const {+ return (type == field_types::OBJECT || type == field_types::OBJECT_ARRAY);+ }+bool is_string() const {return (type == field_types::STRING || type == field_types::STRING_ARRAY);}@@ -128,7 +147,7 @@ struct field {return (type == field_types::STRING_ARRAY || type == field_types::INT32_ARRAY ||type == field_types::FLOAT_ARRAY ||type == field_types::INT64_ARRAY || type == field_types::BOOL_ARRAY ||- type == field_types::GEOPOINT_ARRAY);+ type == field_types::GEOPOINT_ARRAY || type == field_types::OBJECT_ARRAY);}bool is_singular() const {@@ -169,7 +188,8 @@ struct field {}bool has_valid_type() const {- bool is_basic_type = is_string() || is_integer() || is_float() || is_bool() || is_geopoint() || is_auto();+ bool is_basic_type = is_string() || is_integer() || is_float() || is_bool() || is_geopoint() ||+ is_object() || is_auto();if(!is_basic_type) {return field_types::is_string_or_array(type);}@@ -195,10 +215,6 @@ struct field {return true;}- if(obj.is_object()) {- return false;- }-return get_single_type(obj, field_type);}@@ -223,6 +239,11 @@ struct field {return true;}+ if(obj.is_object()) {+ field_type = field_types::OBJECT;+ return true;+ }+return false;}@@ -353,6 +374,16 @@ struct field {return Option<bool>(true);}++ static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,+ const std::string& flat_name, std::vector<field>& flattened_fields);++ static bool flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,+ std::vector<std::string>& path_parts, size_t path_index, bool has_array,+ bool has_obj_array, std::vector<field>& flattened_fields);++ static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,+ std::vector<field>& flattened_fields);};struct filter {@@ -429,7 +460,7 @@ struct filter {NUM_COMPARATOR& num_comparator);static Option<bool> parse_filter_query(const std::string& simple_filter_query,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const Store* store,const std::string& doc_id_prefix,std::vector<filter>& filters);diff --git a/include/index.h b/include/index.hindex 896393c6..1dc7bb7c 100644--- a/include/index.h+++ b/include/index.h@@ -481,7 +481,7 @@ private:size_t num_documents;- std::unordered_map<std::string, field> search_schema;+ tsl::htrie_map<char, field> search_schema;spp::sparse_hash_map<std::string, art_tree*> search_index;@@ -719,7 +719,7 @@ public:const Store* store,SynonymIndex* synonym_index,ThreadPool* thread_pool,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const std::vector<char>& symbols_to_index,const std::vector<char>& token_separators);@@ -768,11 +768,11 @@ public:uint64_t get_distinct_id(const std::vector<std::string>& group_by_fields, const uint32_t seq_id) const;static void compute_token_offsets_facets(index_record& record,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const std::vector<char>& local_token_separators,const std::vector<char>& local_symbols_to_index);- static void scrub_reindex_doc(const std::unordered_map<std::string, field>& search_schema,+ static void scrub_reindex_doc(const tsl::htrie_map<char, field>& search_schema,nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);static void tokenize_string_field(const nlohmann::json& document,@@ -814,7 +814,7 @@ public:static void validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,const size_t batch_start_index, const size_t batch_size,const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,+ const tsl::htrie_map<char, field> & search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,@@ -823,7 +823,7 @@ public:static size_t batch_memory_index(Index *index,std::vector<index_record>& iter_batch,const std::string& default_sorting_field,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,@@ -851,7 +851,7 @@ public:static Option<uint32_t> validate_index_in_memory(nlohmann::json &document, uint32_t seq_id,const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,+ const tsl::htrie_map<char, field> & search_schema,const index_operation_t op,const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values);diff --git a/src/collection.cpp b/src/collection.cppindex 93f72334..fda6cf1c 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2,7 +2,6 @@#include <numeric>#include <chrono>-#include <array_utils.h>#include <match_score.h>#include <string_utils.h>#include <art.h>@@ -20,24 +19,6 @@const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";-struct match_index_t {- Match match;- uint64_t match_score = 0;- size_t index;-- match_index_t(Match match, uint64_t match_score, size_t index): match(match), match_score(match_score),- index(index) {-- }-- bool operator<(const match_index_t& a) const {- if(match_score != a.match_score) {- return match_score > a.match_score;- }- return index < a.index;- }-};-Collection::Collection(const std::string& name, const uint32_t collection_id, const uint64_t created_at,const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,const std::string& default_sorting_field,@@ -287,6 +268,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmaif(index_records.size() == 1) {const auto& rec = index_records[0];document = rec.is_update ? rec.new_doc : rec.doc;+ remove_flat_fields(document);}index_records.clear();batch_doc_ids.clear();@@ -406,19 +388,6 @@ size_t Collection::batch_index_in_memory(std::vector<index_record>& index_recordreturn num_indexed;}-void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash_set<std::string>& include_fields,- const spp::sparse_hash_set<std::string>& exclude_fields) {- auto it = document.begin();- for(; it != document.end(); ) {- if (exclude_fields.count(it.key()) != 0 ||- (!include_fields.empty() && include_fields.count(it.key()) == 0)) {- it = document.erase(it);- } else {- ++it;- }- }-}-void Collection::curate_results(string& actual_query, bool enable_overrides, bool already_segmented,const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,@@ -559,7 +528,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<std::string error = "Bad syntax for sorting field `" + actual_field_name + "`";- if(!field_it->second.is_geopoint()) {+ if(!field_it.value().is_geopoint()) {// check for null value orderconst std::string& sort_params_str = sort_field_std.name.substr(paran_start + 1,sort_field_std.name.size() -@@ -676,7 +645,7 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<if(sort_field_std.name != sort_field_const::text_match) {const auto field_it = search_schema.find(sort_field_std.name);- if(field_it == search_schema.end() || !field_it->second.sort || !field_it->second.index) {+ if(field_it == search_schema.end() || !field_it.value().sort || !field_it.value().index) {std::string error = "Could not find a field named `" + sort_field_std.name +"` in the schema for sorting.";return Option<bool>(404, error);@@ -728,6 +697,33 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<return Option<bool>(true);}+Option<bool> Collection::extract_field_name(const std::string& field_name,+ const tsl::htrie_map<char, field>& search_schema,+ std::vector<std::string>& processed_search_fields) {+ if(search_schema.count(field_name) == 0) {+ // we should check if the field refers to an object, which won't be explicitly be present in the schema+ auto prefix_it = search_schema.equal_prefix_range(field_name);+ bool object_field_found = false;+ for(auto kv = prefix_it.first; kv != prefix_it.second; ++kv) {+ // field_name prefix must be followed by a "." to indicate an object search+ if(kv.key().size() > field_name.size() && kv.key()[field_name.size()] == '.' &&+ kv.value().is_string()) {+ processed_search_fields.push_back(kv.key());+ object_field_found = true;+ }+ }++ if(!object_field_found) {+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";+ return Option<bool>(404, error);+ }+ } else {+ processed_search_fields.push_back(field_name);+ }++ return Option<bool>(true);+}+Option<nlohmann::json> Collection::search(const std::string & raw_query,const std::vector<std::string>& raw_search_fields,const std::string & simple_filter_query, const std::vector<std::string>& facet_fields,@@ -745,7 +741,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,size_t typo_tokens_threshold,const std::string& pinned_hits_str,const std::string& hidden_hits_str,- const std::vector<std::string>& group_by_fields,+ const std::vector<std::string>& raw_group_by_fields,size_t group_limit,const std::string& highlight_start_tag,const std::string& highlight_end_tag,@@ -784,7 +780,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,"number of `query_by` fields.");}- if(!group_by_fields.empty() && (group_limit == 0 || group_limit > GROUP_LIMIT_MAX)) {+ if(!raw_group_by_fields.empty() && (group_limit == 0 || group_limit > GROUP_LIMIT_MAX)) {return Option<nlohmann::json>(400, "Value of `group_limit` must be between 1 and " +std::to_string(GROUP_LIMIT_MAX) + ".");}@@ -810,17 +806,21 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}- if(group_by_fields.empty()) {+ if(raw_group_by_fields.empty()) {group_limit = 0;}// validate search fields- for(const std::string & field_name: raw_search_fields) {- if(search_schema.count(field_name) == 0) {- std::string error = "Could not find a field named `" + field_name + "` in the schema.";- return Option<nlohmann::json>(404, error);+ std::vector<std::string> processed_search_fields;++ for(const std::string& field_name: raw_search_fields) {+ auto field_op = extract_field_name(field_name, search_schema, processed_search_fields);+ if(!field_op.ok()) {+ return Option<nlohmann::json>(404, field_op.error());}+ }+ for(const std::string & field_name: processed_search_fields) {field search_field = search_schema.at(field_name);if(!search_field.index) {@@ -835,6 +835,15 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}// validate group by fields+ std::vector<std::string> group_by_fields;++ for(const std::string& field_name: raw_group_by_fields) {+ auto field_op = extract_field_name(field_name, search_schema, group_by_fields);+ if(!field_op.ok()) {+ return Option<nlohmann::json>(404, field_op.error());+ }+ }+for(const std::string & field_name: group_by_fields) {if(search_schema.count(field_name) == 0) {std::string error = "Could not find a field named `" + field_name + "` in the schema.";@@ -850,12 +859,40 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}+ std::vector<std::string> include_fields_vec;+ std::vector<std::string> exclude_fields_vec;+ tsl::htrie_set<char> include_fields_full;+ tsl::htrie_set<char> exclude_fields_full;++ for(auto& f_name: include_fields) {+ auto field_op = extract_field_name(f_name, search_schema, include_fields_vec);+ if(!field_op.ok()) {+ return Option<nlohmann::json>(404, field_op.error());+ }+ }++ for(auto& f_name: exclude_fields) {+ auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec);+ if(!field_op.ok()) {+ return Option<nlohmann::json>(404, field_op.error());+ }+ }++ for(auto& f_name: include_fields_vec) {+ include_fields_full.insert(f_name);+ }++ for(auto& f_name: exclude_fields_vec) {+ exclude_fields_full.insert(f_name);+ }++// process weights for search fieldsstd::vector<std::string> reordered_search_fields;std::vector<search_field_t> weighted_search_fields;- process_search_field_weights(raw_search_fields, query_by_weights, weighted_search_fields, reordered_search_fields);+ process_search_field_weights(processed_search_fields, query_by_weights, weighted_search_fields, reordered_search_fields);- const std::vector<std::string>& search_fields = reordered_search_fields.empty() ? raw_search_fields+ const std::vector<std::string>& search_fields = reordered_search_fields.empty() ? processed_search_fields: reordered_search_fields;std::vector<facet> facets;@@ -1159,7 +1196,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,std::sort(raw_result_kvs.begin(), raw_result_kvs.end(), Topster::is_greater_kv_group);// restore original scores- for(size_t i = 0; i < max_kvs_bucketed; i++) {+ for(i = 0; i < max_kvs_bucketed; i++) {raw_result_kvs[i][0]->scores[raw_result_kvs[i][0]->match_score_index] = result_scores[i];}}@@ -1232,10 +1269,16 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,// handle which fields have to be highlightedstd::vector<highlight_field_t> highlight_items;+ tsl::htrie_set<char> hfield_names;if(query != "*") {- process_highlight_fields(search_fields, include_fields, exclude_fields, highlight_fields, highlight_full_fields,- infixes, q_tokens, search_params->qtoken_set, highlight_items);+ process_highlight_fields(search_fields, include_fields_full, exclude_fields_full, highlight_fields,+ highlight_full_fields, infixes, q_tokens, search_params->qtoken_set,+ highlight_items);++ for(auto& highlight_item: highlight_items) {+ hfield_names.insert(highlight_item.name);+ }}nlohmann::json result = nlohmann::json::object();@@ -1264,6 +1307,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}nlohmann::json& hits_array = group_limit ? group_hits["hits"] : result["hits"];+ nlohmann::json group_key = nlohmann::json::array();for(const KV* field_order_kv: kv_group) {const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv->key);@@ -1276,6 +1320,14 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,continue;}+ nlohmann::json highlight_doc;++ if(!highlight_items.empty()) {+ highlight_doc = document;+ remove_flat_fields(highlight_doc);+ highlight_doc.erase("id");+ }+nlohmann::json wrapper_doc;wrapper_doc["highlights"] = nlohmann::json::array();std::vector<highlight_t> highlights;@@ -1294,9 +1346,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,search_field.type == field_types::STRING_ARRAY)) {highlight_t highlight;+ highlight.field = search_field.name;highlight_result(raw_query, search_field, i, highlight_item.qtoken_leaves, q_tokens, field_order_kv,- document,string_utils, snippet_threshold, highlight_affix_num_tokens,- highlight_item.fully_highlighted, highlight_item.infix,+ document, highlight_doc, string_utils, snippet_threshold,+ highlight_affix_num_tokens, highlight_item.fully_highlighted, highlight_item.infix,highlight_start_tag, highlight_end_tag, index_symbols, highlight);if(!highlight.snippets.empty()) {highlights.push_back(highlight);@@ -1304,9 +1357,17 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}+ // remove fields from highlight doc that were not highlighted+ prune_doc(highlight_doc, hfield_names, tsl::htrie_set<char>(), "");std::sort(highlights.begin(), highlights.end());for(const auto & highlight: highlights) {+ auto field_it = search_schema.find(highlight.field);+ if(field_it == search_schema.end() || field_it->nested) {+ // nested field highlighting will be available only in the new highlight structure.+ continue;+ }+nlohmann::json h_json = nlohmann::json::object();h_json["field"] = highlight.field;@@ -1330,8 +1391,19 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,//wrapper_doc["seq_id"] = (uint32_t) field_order_kv->key;- prune_document(document, include_fields, exclude_fields);+ if(group_limit && group_key.empty()) {+ for(const auto& field_name: group_by_fields) {+ if(document.count(field_name) != 0) {+ group_key.push_back(document[field_name]);+ }+ }+ }++ prune_doc(document, include_fields_full, exclude_fields_full);+ remove_flat_fields(document);+wrapper_doc["document"] = document;+ wrapper_doc["highlight"] = highlight_doc;if(field_order_kv->match_score_index == CURATED_RECORD_IDENTIFIER) {wrapper_doc["curated"] = true;@@ -1360,15 +1432,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}if(group_limit) {- const auto& document = group_hits["hits"][0]["document"];-- group_hits["group_key"] = nlohmann::json::array();- for(const auto& field_name: group_by_fields) {- if(document.count(field_name) != 0) {- group_hits["group_key"].push_back(document[field_name]);- }- }-+ group_hits["group_key"] = group_key;result["grouped_hits"].push_back(group_hits);}}@@ -1632,8 +1696,8 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_s}void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,- const spp::sparse_hash_set<std::string>& include_fields,- const spp::sparse_hash_set<std::string>& exclude_fields,+ const tsl::htrie_set<char>& include_fields,+ const tsl::htrie_set<char>& exclude_fields,const string& highlight_fields,const std::string& highlight_full_fields,const std::vector<enable_t>& infixes,@@ -1645,7 +1709,12 @@ void Collection::process_highlight_fields(const std::vector<std::string>& searchspp::sparse_hash_set<std::string> fields_highlighted_fully_set;std::vector<std::string> fields_highlighted_fully_vec;StringUtils::split(highlight_full_fields, fields_highlighted_fully_vec, ",");+ std::vector<std::string> fields_highlighted_fully_expanded;for(std::string & highlight_full_field: fields_highlighted_fully_vec) {+ extract_field_name(highlight_full_field, search_schema, fields_highlighted_fully_expanded);+ }++ for(std::string & highlight_full_field: fields_highlighted_fully_expanded) {fields_highlighted_fully_set.insert(highlight_full_field);}@@ -1977,6 +2046,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaconst tsl::htrie_map<char, token_leaf>& qtoken_leaves,const std::vector<std::string>& q_tokens,const KV* field_order_kv, const nlohmann::json & document,+ nlohmann::json& highlight_doc,StringUtils & string_utils,const size_t snippet_threshold,const size_t highlight_affix_num_tokens,@@ -1985,7 +2055,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaconst std::string& highlight_start_tag,const std::string& highlight_end_tag,const uint8_t* index_symbols,- highlight_t & highlight) const {+ highlight_t& highlight) const {if(q_tokens.size() == 1 && q_tokens[0] == "*") {return;@@ -2039,6 +2109,44 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaLOG(INFO) << "Token: " << qtok_buff << ", root_len: " << it.value().root_len;}*/+ /* We have to handle the following differently:++ a) top-level field or a simple nested field+ b) field inside an array of objects++ For a) we can rely on matching index positions since field value on-disk will be in the same order+ as the field value indexed. For b) we need to do a keyword based highlight since we cannot use+ indexed offsets.+ */++ if(search_field.nested_array) {+ std::vector<std::string> path_parts;+ StringUtils::split(search_field.name, path_parts, ".");++ highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, [&](nlohmann::json& str_obj) {+ Match match;+ match_index_t match_index(match, 0, 0);+ int last_valid_offset_index = -1;+ size_t last_valid_offset = 0;++ std::string text = str_obj.get<std::string>();+ bool found_higlight = handle_highlight_text(text, normalise, search_field, symbols_to_index,+ token_separators, highlight, string_utils, is_cyrillic,+ highlight_affix_num_tokens,+ qtoken_leaves, last_valid_offset_index, match,+ last_raw_q_token,+ highlight_fully, snippet_threshold, is_infix_search,+ raw_query_tokens,+ last_valid_offset, highlight_start_tag, highlight_end_tag,+ index_symbols, match_index);+ if(!highlight.snippets.empty()) {+ str_obj = highlight.snippets[0];+ }+ });++ return ;+ }+std::vector<void*> posting_lists;for(auto token_leaf: qtoken_leaves) {posting_lists.push_back(token_leaf.leaf->values);@@ -2113,176 +2221,225 @@ void Collection::highlight_result(const std::string& raw_query, const field &seatext = document[search_field.name][match_index.index];}- Tokenizer tokenizer(text, normalise, false, search_field.locale, symbols_to_index, token_separators);+ handle_highlight_text(text, normalise, search_field, symbols_to_index, token_separators,+ highlight, string_utils, is_cyrillic, highlight_affix_num_tokens,+ qtoken_leaves, last_valid_offset_index, match, last_raw_q_token,+ highlight_fully, snippet_threshold, is_infix_search, raw_query_tokens,+ last_valid_offset, highlight_start_tag, highlight_end_tag,+ index_symbols, match_index);- // word tokenizer is a secondary tokenizer used for specific languages that requires transliteration- Tokenizer word_tokenizer("", true, false, search_field.locale, symbols_to_index, token_separators);-- if(search_field.locale == "ko") {- text = string_utils.unicode_nfkd(text);- }-- // need an ordered map here to ensure that it is ordered by the key (start offset)- std::map<size_t, size_t> token_offsets;+ }- int match_offset_index = 0;- std::string raw_token;- std::set<std::string> token_hits; // used to identify repeating tokens- size_t raw_token_index = 0, tok_start = 0, tok_end = 0;+ highlight.field = search_field.name;+ highlight.field_index = search_field_index;- // based on `highlight_affix_num_tokens`- size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);+ if(!match_indices.empty()) {+ highlight.match_score = match_indices[0].match_score;+ }- // window used to locate the starting offset for snippet on the text- std::list<size_t> snippet_start_window;+ if(search_field.nested) {+ std::vector<std::string> parts;+ StringUtils::split(search_field.name, parts, ".");+ nlohmann::json* val = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;- highlight.matched_tokens.emplace_back();- std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();- bool found_first_match = false;+ for(size_t i = 1; val != nullptr && i < parts.size(); i++) {+ const auto& part = parts[i];+ if(val->contains(part)) {+ val = &val->at(part);+ } else {+ val = nullptr;+ }+ }- while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {- if(is_cyrillic) {- bool found_token = word_tokenizer.tokenize(raw_token);- if(!found_token) {- continue;+ if(val) {+ if(highlight.indices.empty()) {+ *val = highlight.snippets[0];+ } else {+ if(val->is_array()) {+ for(size_t hi = 0; hi < highlight.indices.size(); hi++) {+ val->at(highlight.indices[hi]) = highlight.snippets[hi];+ }}}+ }+ }+}- if(!found_first_match) {- if(snippet_start_window.size() == highlight_affix_num_tokens + 1) {- snippet_start_window.pop_front();- }+bool Collection::handle_highlight_text(std::string& text, bool normalise, const field &search_field,+ const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,+ highlight_t& highlight, StringUtils & string_utils, bool is_cyrillic,+ const size_t highlight_affix_num_tokens,+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves,+ int last_valid_offset_index, const Match& match,+ const std::string& last_raw_q_token, bool highlight_fully, const size_t snippet_threshold,+ bool is_infix_search, std::vector<std::string>& raw_query_tokens, size_t last_valid_offset,+ const std::string& highlight_start_tag, const std::string& highlight_end_tag,+ const uint8_t* index_symbols, const match_index_t& match_index) const {- snippet_start_window.push_back(tok_start);- }+ Tokenizer tokenizer(text, normalise, false, search_field.locale, symbols_to_index, token_separators);- bool token_already_found = (token_hits.find(raw_token) != token_hits.end());- auto qtoken_it = qtoken_leaves.find(raw_token);+ // word tokenizer is a secondary tokenizer used for specific languages that requires transliteration+ Tokenizer word_tokenizer("", true, false, search_field.locale, symbols_to_index, token_separators);- // ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestion- if ((found_first_match && token_already_found) ||- (match_offset_index <= last_valid_offset_index &&- match.offsets[match_offset_index].offset == raw_token_index)) {+ if(search_field.locale == "ko") {+ text = string_utils.unicode_nfkd(text);+ }- // check if the matched token is a prefix of this found token- if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().is_prefix &&- qtoken_it.value().root_len < raw_token.size()) {- // need to ensure that only the prefix portion is highlighted- // if length diff is within 2, we still might not want to highlight partially in some cases- // e.g. "samsng" vs "samsung" -> full highlight is preferred, unless it's a full prefix match- //size_t char_diff = raw_token.size() - qtoken_it.value().root_len;- size_t char_diff = (tok_end - tok_start + 1) - last_raw_q_token.size();- auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ?- tok_end : (tok_end - char_diff);- token_offsets.emplace(tok_start, new_tok_end);- } else {- token_offsets.emplace(tok_start, tok_end);- }+ // need an ordered map here to ensure that it is ordered by the key (start offset)+ std::map<size_t, size_t> token_offsets;- token_hits.insert(raw_token);+ int match_offset_index = 0;+ std::string raw_token;+ std::set<std::string> token_hits; // used to identify repeating tokens+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;- // to skip over duplicate tokens in the query- do {- match_offset_index++;- } while(match_offset_index <= last_valid_offset_index &&- match.offsets[match_offset_index - 1].offset == match.offsets[match_offset_index].offset);+ // based on `highlight_affix_num_tokens`+ size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);- if(!found_first_match) {- snippet_start_offset = snippet_start_window.front();- }+ // window used to locate the starting offset for snippet on the text+ std::list<size_t> snippet_start_window;++ highlight.matched_tokens.emplace_back();+ std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();+ bool found_first_match = false;++ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ if(is_cyrillic) {+ bool found_token = word_tokenizer.tokenize(raw_token);+ if(!found_token) {+ continue;+ }+ }- found_first_match = true;+ if(!found_first_match) {+ if(snippet_start_window.size() == highlight_affix_num_tokens + 1) {+ snippet_start_window.pop_front();+ }- } else if((highlight_fully || text.size() < snippet_threshold * 6) &&- qtoken_leaves.find(raw_token) != qtoken_leaves.end()) {+ snippet_start_window.push_back(tok_start);+ }- // Token might not appear in the best matched window, which is limited to a size of 10.- // If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will- // locate all tokens that appear in the query / query candidates+ bool token_already_found = (token_hits.find(raw_token) != token_hits.end());+ auto qtoken_it = qtoken_leaves.find(raw_token);- if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().is_prefix &&- qtoken_it.value().root_len < raw_token.size()) {- // need to ensure that only the prefix portion is highlighted- size_t char_diff = (tok_end - tok_start + 1) - last_raw_q_token.size();- auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ?- tok_end : (tok_end - char_diff);- token_offsets.emplace(tok_start, new_tok_end);- } else {- token_offsets.emplace(tok_start, tok_end);- }+ // ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestion+ if ((found_first_match && token_already_found) ||+ (match_offset_index <= last_valid_offset_index &&+ match.offsets[match_offset_index].offset == raw_token_index)) {- token_hits.insert(raw_token);- } else if(is_infix_search && text.size() < 100 &&- raw_token.find(raw_query_tokens.front()) != std::string::npos) {+ // check if the matched token is a prefix of this found token+ if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().is_prefix &&+ qtoken_it.value().root_len < raw_token.size()) {+ // need to ensure that only the prefix portion is highlighted+ // if length diff is within 2, we still might not want to highlight partially in some cases+ // e.g. "samsng" vs "samsung" -> full highlight is preferred, unless it's a full prefix match+ //size_t char_diff = raw_token.size() - qtoken_it.value().root_len;+ size_t char_diff = (tok_end - tok_start + 1) - last_raw_q_token.size();+ auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ?+ tok_end : (tok_end - char_diff);+ token_offsets.emplace(tok_start, new_tok_end);+ } else {token_offsets.emplace(tok_start, tok_end);- token_hits.insert(raw_token);}- if(raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {- // register end of highlight snippet- if(snippet_end_offset == text.size() - 1) {- snippet_end_offset = tok_end;- }+ token_hits.insert(raw_token);++ // to skip over duplicate tokens in the query+ do {+ match_offset_index++;+ } while(match_offset_index <= last_valid_offset_index &&+ match.offsets[match_offset_index - 1].offset == match.offsets[match_offset_index].offset);++ if(!found_first_match) {+ snippet_start_offset = snippet_start_window.front();}- // We can break early only if we have:- // a) run out of matched indices- // b) token_index exceeds the suffix tokens boundary- // c) raw_token_index exceeds snippet threshold- // d) highlight fully is not requested+ found_first_match = true;- if(raw_token_index >= snippet_threshold &&- match_offset_index > last_valid_offset_index &&- raw_token_index >= last_valid_offset + highlight_affix_num_tokens &&- !highlight_fully) {- break;+ } else if((highlight_fully || text.size() < snippet_threshold * 6) &&+ qtoken_leaves.find(raw_token) != qtoken_leaves.end()) {++ // Token might not appear in the best matched window, which is limited to a size of 10.+ // If field is marked to be highlighted fully, or field length exceeds snippet_threshold, we will+ // locate all tokens that appear in the query / query candidates++ if(qtoken_it != qtoken_leaves.end() && qtoken_it.value().is_prefix &&+ qtoken_it.value().root_len < raw_token.size()) {+ // need to ensure that only the prefix portion is highlighted+ size_t char_diff = (tok_end - tok_start + 1) - last_raw_q_token.size();+ auto new_tok_end = (char_diff <= 2 && qtoken_it.value().num_typos != 0) ?+ tok_end : (tok_end - char_diff);+ token_offsets.emplace(tok_start, new_tok_end);+ } else {+ token_offsets.emplace(tok_start, tok_end);}- }- if(token_offsets.empty()) {- continue;+ token_hits.insert(raw_token);+ } else if(is_infix_search && text.size() < 100 &&+ raw_token.find(raw_query_tokens.front()) != std::string::npos) {+ token_offsets.emplace(tok_start, tok_end);+ token_hits.insert(raw_token);}- if(raw_token_index <= snippet_threshold-1) {- // fully highlight field whose token size is less than given snippet threshold- snippet_start_offset = 0;- snippet_end_offset = text.size() - 1;+ if(raw_token_index >= last_valid_offset + highlight_affix_num_tokens) {+ // register end of highlight snippet+ if(snippet_end_offset == text.size() - 1) {+ snippet_end_offset = tok_end;+ }}- // `token_offsets` has a list of ranges to target for highlighting- // tokens from query might occur before actual snippet start offset: we skip that- auto offset_it = token_offsets.begin();- while(offset_it != token_offsets.end() && offset_it->first < snippet_start_offset) {- offset_it++;+ // We can break early only if we have:+ // a) run out of matched indices+ // b) token_index exceeds the suffix tokens boundary+ // c) raw_token_index exceeds snippet threshold+ // d) highlight fully is not requested++ if(raw_token_index >= snippet_threshold &&+ match_offset_index > last_valid_offset_index &&+ raw_token_index >= last_valid_offset + highlight_affix_num_tokens &&+ !highlight_fully) {+ break;}+ }- std::stringstream highlighted_text;- highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,- snippet_end_offset, matched_tokens, offset_it,- highlighted_text, index_symbols, snippet_start_offset);+ if(token_offsets.empty()) {+ return false;+ }- highlight.snippets.push_back(highlighted_text.str());- if(search_field.type == field_types::STRING_ARRAY) {- highlight.indices.push_back(match_index.index);- }+ if(raw_token_index <= snippet_threshold-1) {+ // fully highlight field whose token size is less than given snippet threshold+ snippet_start_offset = 0;+ snippet_end_offset = text.size() - 1;+ }- if(highlight_fully) {- std::stringstream value_stream;- offset_it = token_offsets.begin();- std::vector<std::string> full_matched_tokens;- highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,- text.size()-1, full_matched_tokens, offset_it,- value_stream, index_symbols, 0);- highlight.values.push_back(value_stream.str());- }+ // `token_offsets` has a list of ranges to target for highlighting+ // tokens from query might occur before actual snippet start offset: we skip that+ auto offset_it = token_offsets.begin();+ while(offset_it != token_offsets.end() && offset_it->first < snippet_start_offset) {+ offset_it++;}- highlight.field = search_field.name;- highlight.field_index = search_field_index;+ std::stringstream highlighted_text;+ highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,+ snippet_end_offset, matched_tokens, offset_it,+ highlighted_text, index_symbols, snippet_start_offset);- if(!match_indices.empty()) {- highlight.match_score = match_indices[0].match_score;+ highlight.snippets.push_back(highlighted_text.str());+ if(search_field.type == field_types::STRING_ARRAY) {+ highlight.indices.push_back(match_index.index);+ }++ if(highlight_fully) {+ std::stringstream value_stream;+ offset_it = token_offsets.begin();+ std::vector<std::string> full_matched_tokens;+ highlight_text(highlight_start_tag, highlight_end_tag, text, token_offsets,+ text.size()-1, full_matched_tokens, offset_it,+ value_stream, index_symbols, 0);+ highlight.values.push_back(value_stream.str());}++ return true;}void Collection::highlight_text(const string& highlight_start_tag, const string& highlight_end_tag,@@ -2528,8 +2685,8 @@ std::vector<std::string> Collection::get_facet_fields() {std::vector<std::string> facet_fields_copy;for(auto it = search_schema.begin(); it != search_schema.end(); ++it) {- if(it->second.facet) {- facet_fields_copy.push_back(it->first);+ if(it.value().facet) {+ facet_fields_copy.push_back(it.key());}}@@ -2541,8 +2698,8 @@ std::vector<field> Collection::get_sort_fields() {std::vector<field> sort_fields_copy;for(auto it = search_schema.begin(); it != search_schema.end(); ++it) {- if(it->second.sort) {- sort_fields_copy.push_back(it->second);+ if(it.value().sort) {+ sort_fields_copy.push_back(it.value());}}@@ -2559,7 +2716,7 @@ std::unordered_map<std::string, field> Collection::get_dynamic_fields() {return dynamic_fields;}-std::unordered_map<std::string, field> Collection::get_schema() {+tsl::htrie_map<char, field> Collection::get_schema() {std::shared_lock lock(mutex);return search_schema;};@@ -2724,7 +2881,7 @@ Option<bool> Collection::persist_collection_meta() {return Option<bool>(true);}-Option<bool> Collection::batch_alter_data(const std::unordered_map<std::string, field>& schema_additions,+Option<bool> Collection::batch_alter_data(const tsl::htrie_map<char, field>& schema_additions,const std::unordered_map<std::string, field>& new_dynamic_fields,const std::vector<field>& del_fields,const std::string& this_fallback_field_type,@@ -2732,15 +2889,13 @@ Option<bool> Collection::batch_alter_data(const std::unordered_map<std::string,// Update schema with additions (deletions can only be made later)std::vector<field> new_fields;- for(auto& kv: schema_additions) {- const auto& f = kv.second;-+ for(auto& f: schema_additions) {if(f.name == ".*") {fields.push_back(f);continue;}- search_schema.emplace(kv.first, f);+ search_schema.emplace(f.name, f);new_fields.push_back(f);fields.push_back(f);}@@ -2849,8 +3004,8 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {std::unique_lock lock(mutex);// Validate that all stored documents are compatible with the proposed schema changes.- std::unordered_map<std::string, field> schema_additions;- std::unordered_map<std::string, field> schema_reindex;+ tsl::htrie_map<char, field> schema_additions;+ tsl::htrie_map<char, field> schema_reindex;std::unordered_map<std::string, field> addition_dynamic_fields;std::unordered_map<std::string, field> reindex_dynamic_fields;std::vector<field> del_fields;@@ -2895,9 +3050,99 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {return Option<bool>(true);}+void Collection::remove_flat_fields(nlohmann::json& document) {+ if(document.contains(".flat")) {+ for(const auto& flat_key: document[".flat"].get<std::vector<std::string>>()) {+ document.erase(flat_key);+ }+ document.erase(".flat");+ }+}++void Collection::prune_doc(nlohmann::json& doc,+ const tsl::htrie_set<char>& include_names,+ const tsl::htrie_set<char>& exclude_names,+ std::string parent_name, size_t depth) {+ // doc can only be an object+ auto it = doc.begin();+ while(it != doc.end()) {+ std::string nested_name = parent_name + (parent_name.empty() ? it.key() : "." + it.key());++ //LOG(INFO) << "it.key(): " << it.key() << ", nested_name: " << nested_name;++ // use prefix lookup to prune non-matching sub-trees early+ auto prefix_it = include_names.equal_prefix_range(nested_name);+ if(!include_names.empty() && prefix_it.first == prefix_it.second) {+ // prefix not found in allowed list of highlight field names, so can trim early+ it = doc.erase(it);+ continue ;+ }++ if(exclude_names.count(nested_name) != 0) {+ it = doc.erase(it);+ continue ;+ }++ if(exclude_names.empty() && !include_names.empty() && include_names.count(nested_name) != 0) {+ // without exclusions, we can pick the sub-tree early if parent name is found in include names+ it++;+ continue;+ }++ if(it.value().is_object()) {+ prune_doc(it.value(), include_names, exclude_names, nested_name, depth+1);+ if(it.value().empty()) {+ it = doc.erase(it);+ } else {+ it++;+ }++ continue;+ }++ else if(it.value().is_array()) {+ bool orig_array_empty = it.value().empty();+ bool primitive_array = true;+ auto arr_it = it.value().begin();+ while(arr_it != it.value().end()) {+ // NOTE: we will not support array of array of nested objects+ primitive_array = primitive_array && !arr_it.value().is_object();+ if(arr_it.value().is_object()) {+ prune_doc(arr_it.value(), include_names, exclude_names, nested_name, depth+1);+ if(arr_it.value().empty()) {+ arr_it = it.value().erase(arr_it);+ continue;+ }+ }++ arr_it++;+ }++ if(!orig_array_empty && it.value().empty()) {+ // only drop field if array became empty because of pruning (and not empty already)+ it = doc.erase(it);+ continue;+ }++ if(!primitive_array) {+ it++;+ continue;+ }+ }++ if(!include_names.empty() && include_names.count(nested_name) == 0) {+ // at this point, name should match fully, otherwise we should erase the value+ it = doc.erase(it);+ continue;+ }++ it++;+ }+}+Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,- std::unordered_map<std::string, field>& schema_additions,- std::unordered_map<std::string, field>& schema_reindex,+ tsl::htrie_map<char, field>& schema_additions,+ tsl::htrie_map<char, field>& schema_reindex,std::unordered_map<std::string, field>& addition_dynamic_fields,std::unordered_map<std::string, field>& reindex_dynamic_fields,std::vector<field>& del_fields,@@ -2919,7 +3164,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,// basic validation of fieldsstd::vector<field> diff_fields;- std::unordered_map<std::string, field> updated_search_schema = search_schema;+ tsl::htrie_map<char, field> updated_search_schema = search_schema;size_t num_auto_detect_fields = 0;// since fields can be deleted and added in the same change set,@@ -2961,7 +3206,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,auto dyn_field_it = dynamic_fields.find(field_name);auto found_dyn_field = (dyn_field_it != dynamic_fields.end()) ||- (found_field && field_types::is_string_or_array(field_it->second.type));+ (found_field && field_types::is_string_or_array(field_it.value().type));if(kv.value().contains("drop")) {if(!kv.value()["drop"].is_boolean() || !kv.value()["drop"].get<bool>()) {@@ -2978,12 +3223,12 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}if(found_field) {- del_fields.push_back(field_it->second);- updated_search_schema.erase(field_it->first);+ del_fields.push_back(field_it.value());+ updated_search_schema.erase(field_it.key());// we will also have to resolve the dynamic field names which match the static field namefor(auto& field_kv: dynamic_fields) {- if(std::regex_match(field_kv.first, std::regex(field_it->first))) {+ if(std::regex_match(field_kv.first, std::regex(field_it.key()))) {del_fields.push_back(field_kv.second);}}@@ -2992,12 +3237,12 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,else if(found_dyn_field) {del_fields.push_back(dyn_field_it->second);// we will also have to resolve the actual field names which match the dynamic field pattern- for(auto& field_kv: search_schema) {- if(std::regex_match(field_kv.first, std::regex(dyn_field_it->first))) {- del_fields.push_back(field_kv.second);+ for(auto& a_field: search_schema) {+ if(std::regex_match(a_field.name, std::regex(dyn_field_it->first))) {+ del_fields.push_back(a_field);// if schema contains explicit fields that match dynamic field that're going to be removed,// we will have to remove them from the schema so that validation can occur properly- updated_search_schema.erase(field_kv.first);+ updated_search_schema.erase(a_field.name);}}}@@ -3145,10 +3390,12 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,Option<bool> Collection::detect_new_fields(nlohmann::json& document,const DIRTY_VALUES& dirty_values,- const std::unordered_map<std::string, field>& schema,+ const tsl::htrie_map<char, field>& schema,const std::unordered_map<std::string, field>& dyn_fields,const std::string& fallback_field_type,std::vector<field>& new_fields) {+ std::vector<field> nested_fields;+auto kv = document.begin();while(kv != document.end()) {// we will not index the special "id" key@@ -3265,12 +3512,28 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,new_field.sort = true;}- new_fields.emplace_back(new_field);+ if(new_field.is_object()) {+ nested_fields.push_back(new_field);+ } else {+ new_fields.emplace_back(new_field);+ }}kv++;}+ std::vector<field> flattened_fields;+ auto flatten_op = field::flatten_doc(document, nested_fields, flattened_fields);+ if(!flatten_op.ok()) {+ return flatten_op;+ }++ for(const auto& flattened_field: flattened_fields) {+ if(schema.count(flattened_field.name) == 0) {+ new_fields.push_back(flattened_field);+ }+ }+return Option<bool>(true);}diff --git a/src/field.cpp b/src/field.cppindex 5835a981..f5929894 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -73,7 +73,7 @@ Option<bool> filter::parse_geopoint_filter_value(std::string& raw_value,}Option<bool> filter::parse_filter_query(const string& simple_filter_query,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const Store* store,const std::string& doc_id_prefix,std::vector<filter>& filters) {@@ -522,3 +522,146 @@ Option<bool> field::json_field_to_field(nlohmann::json& field_json, std::vector<return Option<bool>(true);}++bool field::flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,+ const std::string& flat_name, std::vector<field>& flattened_fields) {+ if(value.is_object()) {+ has_obj_array = has_array;+ for(const auto& kv: value.items()) {+ flatten_obj(doc, kv.value(), has_array, has_obj_array, flat_name + "." + kv.key(), flattened_fields);+ }+ } else if(value.is_array()) {+ for(const auto& kv: value.items()) {+ flatten_obj(doc, kv.value(), true, has_obj_array, flat_name, flattened_fields);+ }+ } else {+ // must be a primitive+ if(has_array) {+ doc[flat_name].push_back(value);+ } else {+ doc[flat_name] = value;+ }++ std::string detected_type;+ if(!field::get_type(value, detected_type)) {+ return false;+ }++ if(std::isalnum(detected_type.back()) && has_array) {+ // convert singular type to multi valued type+ detected_type += "[]";+ }++ field flattened_field(flat_name, detected_type, false, true);+ flattened_field.nested = true;+ flattened_field.nested_array = has_obj_array;+ flattened_fields.push_back(flattened_field);+ }++ return true;+}++bool field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, const field& the_field,+ std::vector<std::string>& path_parts, size_t path_index,+ bool has_array, bool has_obj_array, std::vector<field>& flattened_fields) {+ if(path_index == path_parts.size()) {+ // end of path: check if obj matches expected type+ std::string detected_type;+ if(!field::get_type(obj, detected_type)) {+ return false;+ }++ if(std::isalnum(detected_type.back()) && has_array) {+ // convert singular type to multi valued type+ detected_type += "[]";+ }++ has_obj_array = has_obj_array || ((detected_type == field_types::OBJECT) && has_array);++ if(detected_type == the_field.type) {+ if(the_field.is_object()) {+ flatten_obj(doc, obj, has_array, has_obj_array, the_field.name, flattened_fields);+ } else {+ if(has_array) {+ doc[the_field.name].push_back(obj);+ } else {+ doc[the_field.name] = obj;+ }++ field flattened_field(the_field.name, detected_type, false, true);+ flattened_field.nested = (path_index > 1);+ flattened_field.nested_array = has_obj_array;+ flattened_fields.push_back(flattened_field);+ }++ return true;+ }++ // handle differences in detection of numerical types+ bool is_numericaly_valid = (detected_type == field_types::INT64 && (the_field.type == field_types::INT32 ||+ the_field.type == field_types::FLOAT)) ||+ (detected_type == field_types::INT64_ARRAY &&+ (the_field.type == field_types::INT32_ARRAY ||+ the_field.type == field_types::FLOAT_ARRAY));++ if(is_numericaly_valid) {+ if(has_array) {+ doc[the_field.name].push_back(obj);+ } else {+ doc[the_field.name] = obj;+ }++ field flattened_field(the_field.name, the_field.type, false, true);+ flattened_field.nested = (path_index > 1);+ flattened_field.nested_array = has_obj_array;+ flattened_fields.push_back(flattened_field);+ return true;+ } else {+ return false;+ }+ }++ const std::string& fragment = path_parts[path_index];+ const auto& it = obj.find(fragment);++ if(it != obj.end()) {+ if(it.value().is_array()) {+ has_array = true;+ bool resolved = false;+ for(auto& ele: it.value()) {+ has_obj_array = has_obj_array || ele.is_object();+ resolved |= flatten_field(doc, ele, the_field, path_parts, path_index + 1, has_array, has_obj_array, flattened_fields);+ }+ return resolved;+ } else {+ return flatten_field(doc, it.value(), the_field, path_parts, path_index + 1, has_array, has_obj_array, flattened_fields);+ }+ } {+ return false;+ }+}++Option<bool> field::flatten_doc(nlohmann::json& document,+ const std::vector<field>& nested_fields,+ std::vector<field>& flattened_fields) {++ for(auto& nested_field: nested_fields) {+ std::vector<std::string> field_parts;+ StringUtils::split(nested_field.name, field_parts, ".");++ bool resolved = flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields);+ if(!resolved) {+ return Option<bool>(400, "Field `" + nested_field.name + "` was not found or has an incorrect type.");+ }+ }++ std::sort(flattened_fields.begin(), flattened_fields.end());+ flattened_fields.erase(std::unique(flattened_fields.begin(), flattened_fields.end()), flattened_fields.end());++ document[".flat"] = nlohmann::json::array();+ for(auto& f: flattened_fields) {+ document[".flat"].push_back(f.name);+ }++ return Option<bool>(true);+}diff --git a/src/index.cpp b/src/index.cppindex 0947b444..71c73de1 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -52,68 +52,68 @@ struct token_posting_t {Index::Index(const std::string& name, const uint32_t collection_id, const Store* store,SynonymIndex* synonym_index, ThreadPool* thread_pool,- const std::unordered_map<std::string, field> & search_schema,+ const tsl::htrie_map<char, field> & search_schema,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators):name(name), collection_id(collection_id), store(store), synonym_index(synonym_index), thread_pool(thread_pool),search_schema(search_schema),seq_ids(new id_list_t(256)), symbols_to_index(symbols_to_index), token_separators(token_separators) {- for(const auto & fname_field: search_schema) {- if(!fname_field.second.index) {+ for(const auto& a_field: search_schema) {+ if(!a_field.index) {continue;}- if(fname_field.second.is_string()) {+ if(a_field.is_string()) {art_tree *t = new art_tree;art_tree_init(t);- search_index.emplace(fname_field.first, t);- } else if(fname_field.second.is_geopoint()) {+ search_index.emplace(a_field.name, t);+ } else if(a_field.is_geopoint()) {auto field_geo_index = new spp::sparse_hash_map<std::string, std::vector<uint32_t>>();- geopoint_index.emplace(fname_field.first, field_geo_index);+ geopoint_index.emplace(a_field.name, field_geo_index);- if(!fname_field.second.is_single_geopoint()) {+ if(!a_field.is_single_geopoint()) {spp::sparse_hash_map<uint32_t, int64_t*> * doc_to_geos = new spp::sparse_hash_map<uint32_t, int64_t*>();- geo_array_index.emplace(fname_field.first, doc_to_geos);+ geo_array_index.emplace(a_field.name, doc_to_geos);}} else {num_tree_t* num_tree = new num_tree_t;- numerical_index.emplace(fname_field.first, num_tree);+ numerical_index.emplace(a_field.name, num_tree);}- if(fname_field.second.sort) {- if(fname_field.second.type == field_types::STRING) {+ if(a_field.sort) {+ if(a_field.type == field_types::STRING) {adi_tree_t* tree = new adi_tree_t();- str_sort_index.emplace(fname_field.first, tree);- } else if(fname_field.second.type != field_types::GEOPOINT_ARRAY) {+ str_sort_index.emplace(a_field.name, tree);+ } else if(a_field.type != field_types::GEOPOINT_ARRAY) {spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();- sort_index.emplace(fname_field.first, doc_to_score);+ sort_index.emplace(a_field.name, doc_to_score);}}- if(fname_field.second.facet) {+ if(a_field.facet) {array_mapped_facet_t facet_array;for(size_t i = 0; i < ARRAY_FACET_DIM; i++) {facet_array[i] = new facet_map_t();}- facet_index_v3.emplace(fname_field.first, facet_array);+ facet_index_v3.emplace(a_field.name, facet_array);}// initialize for non-string facet fields- if(fname_field.second.facet && !fname_field.second.is_string()) {+ if(a_field.facet && !a_field.is_string()) {art_tree *ft = new art_tree;art_tree_init(ft);- search_index.emplace(fname_field.second.faceted_name(), ft);+ search_index.emplace(a_field.faceted_name(), ft);}- if(fname_field.second.infix) {+ if(a_field.infix) {array_mapped_infix_t infix_sets(ARRAY_INFIX_DIM);for(auto& infix_set: infix_sets) {infix_set = new tsl::htrie_set<char>();}- infix_index.emplace(fname_field.second.name, infix_sets);+ infix_index.emplace(a_field.name, infix_sets);}}@@ -221,15 +221,15 @@ int64_t Index::float_to_in64_t(float f) {}void Index::compute_token_offsets_facets(index_record& record,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const std::vector<char>& local_token_separators,const std::vector<char>& local_symbols_to_index) {const auto& document = record.doc;- for(const auto& field_pair: search_schema) {- const std::string& field_name = field_pair.first;- if(document.count(field_name) == 0 || !field_pair.second.index) {+ for(const auto& the_field: search_schema) {+ const std::string& field_name = the_field.name;+ if(document.count(field_name) == 0 || !the_field.index) {continue;}@@ -238,58 +238,58 @@ void Index::compute_token_offsets_facets(index_record& record,bool is_facet = search_schema.at(field_name).facet;// non-string, non-geo faceted field should be indexed as faceted string field as well- if(field_pair.second.facet && !field_pair.second.is_string() && !field_pair.second.is_geopoint()) {- if(field_pair.second.is_array()) {+ if(the_field.facet && !the_field.is_string() && !the_field.is_geopoint()) {+ if(the_field.is_array()) {std::vector<std::string> strings;- if(field_pair.second.type == field_types::INT32_ARRAY) {+ if(the_field.type == field_types::INT32_ARRAY) {for(int32_t value: document[field_name]){strings.push_back(std::to_string(value));}- } else if(field_pair.second.type == field_types::INT64_ARRAY) {+ } else if(the_field.type == field_types::INT64_ARRAY) {for(int64_t value: document[field_name]){strings.push_back(std::to_string(value));}- } else if(field_pair.second.type == field_types::FLOAT_ARRAY) {+ } else if(the_field.type == field_types::FLOAT_ARRAY) {for(float value: document[field_name]){strings.push_back(StringUtils::float_to_str(value));}- } else if(field_pair.second.type == field_types::BOOL_ARRAY) {+ } else if(the_field.type == field_types::BOOL_ARRAY) {for(bool value: document[field_name]){strings.push_back(std::to_string(value));}}- tokenize_string_array_with_facets(strings, is_facet, field_pair.second,+ tokenize_string_array_with_facets(strings, is_facet, the_field,local_symbols_to_index, local_token_separators,offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);} else {std::string text;- if(field_pair.second.type == field_types::INT32) {+ if(the_field.type == field_types::INT32) {text = std::to_string(document[field_name].get<int32_t>());- } else if(field_pair.second.type == field_types::INT64) {+ } else if(the_field.type == field_types::INT64) {text = std::to_string(document[field_name].get<int64_t>());- } else if(field_pair.second.type == field_types::FLOAT) {+ } else if(the_field.type == field_types::FLOAT) {text = StringUtils::float_to_str(document[field_name].get<float>());- } else if(field_pair.second.type == field_types::BOOL) {+ } else if(the_field.type == field_types::BOOL) {text = std::to_string(document[field_name].get<bool>());}- tokenize_string_with_facets(text, is_facet, field_pair.second,+ tokenize_string_with_facets(text, is_facet, the_field,local_symbols_to_index, local_token_separators,offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);}}- if(field_pair.second.is_string()) {+ if(the_field.is_string()) {- if(field_pair.second.type == field_types::STRING) {- tokenize_string_with_facets(document[field_name], is_facet, field_pair.second,+ if(the_field.type == field_types::STRING) {+ tokenize_string_with_facets(document[field_name], is_facet, the_field,local_symbols_to_index, local_token_separators,offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);} else {- tokenize_string_array_with_facets(document[field_name], is_facet, field_pair.second,+ tokenize_string_array_with_facets(document[field_name], is_facet, the_field,local_symbols_to_index, local_token_separators,offset_facet_hashes.offsets, offset_facet_hashes.facet_hashes);}@@ -303,7 +303,7 @@ void Index::compute_token_offsets_facets(index_record& record,Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint32_t seq_id,const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,+ const tsl::htrie_map<char, field> & search_schema,const index_operation_t op,const std::string& fallback_field_type,const DIRTY_VALUES& dirty_values) {@@ -315,9 +315,8 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3"but is not found in the document.");}- for(const auto& field_pair: search_schema) {- const std::string& field_name = field_pair.first;- const field& a_field = field_pair.second;+ for(const auto& a_field: search_schema) {+ const std::string& field_name = a_field.name;if(field_name == "id") {continue;@@ -454,7 +453,7 @@ Option<uint32_t> Index::validate_index_in_memory(nlohmann::json& document, uint3void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iter_batch,const size_t batch_start_index, const size_t batch_size,const std::string& default_sorting_field,- const std::unordered_map<std::string, field>& search_schema,+ const tsl::htrie_map<char, field>& search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,@@ -527,7 +526,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itesize_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_batch,const std::string & default_sorting_field,- const std::unordered_map<std::string, field> & search_schema,+ const tsl::htrie_map<char, field> & search_schema,const std::string& fallback_field_type,const std::vector<char>& token_separators,const std::vector<char>& symbols_to_index,@@ -2144,7 +2143,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&continue;}- search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it->second, field_name,+ search_field(0, window_tokens, nullptr, 0, num_toks_dropped, field_it.value(), field_name,nullptr, 0, {}, {}, -1, 0, searched_queries, topster, groups_processed,&result_ids, result_ids_len, field_num_results, 0, group_by_fields,false, 4, query_hashes, token_order, false, 0, 0, false, -1, 3, 7, 4);@@ -2167,7 +2166,7 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&uint32_t* exact_strt_ids = new uint32_t[result_ids_len];size_t exact_strt_size = 0;- posting_t::get_exact_matches(posting_lists, field_it->second.is_array(), result_ids, result_ids_len,+ posting_t::get_exact_matches(posting_lists, field_it.value().is_array(), result_ids, result_ids_len,exact_strt_ids, exact_strt_size);delete [] result_ids;@@ -4664,7 +4663,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, constreturn;}- const auto& search_field = search_field_it->second;+ const auto& search_field = search_field_it.value();if(!search_field.index) {return;@@ -4828,7 +4827,6 @@ void Index::tokenize_string_field(const nlohmann::json& document, const field& s}}-// FIXME: returning raw value is a problem!art_leaf* Index::get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len) {std::shared_lock lock(mutex);const art_tree *t = search_index.at(field_name);@@ -5376,7 +5374,7 @@ void Index::get_doc_changes(const index_operation_t op, nlohmann::json& update_d}}-void Index::scrub_reindex_doc(const std::unordered_map<std::string, field>& search_schema,+void Index::scrub_reindex_doc(const tsl::htrie_map<char, field>& search_schema,nlohmann::json& update_doc,nlohmann::json& del_doc,const nlohmann::json& old_doc) {@@ -5393,7 +5391,7 @@ void Index::scrub_reindex_doc(const std::unordered_map<std::string, field>& searcontinue;}- const auto search_field = search_field_it->second; // copy, don't use reference!+ const auto search_field = search_field_it.value(); // copy, don't use reference!// compare values between old and update docs:// if they match, we will remove them from both del and update docsdiff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 3d4c56cf..fdef4ebf 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -409,6 +409,7 @@ TEST_F(CollectionAllFieldsTest, StringifyAllValues) {ASSERT_EQ("Field `int_values` must be an array of string.", add_op.error());// singular field coercion+ doc["int_values"] = {"100"};doc["single_int"] = 100;doc["title"] = "FOURTH";@@ -1108,9 +1109,15 @@ TEST_F(CollectionAllFieldsTest, WildcardFieldAndDictionaryField) {ASSERT_EQ(1, results["hits"].size());auto schema = coll1->get_fields();- ASSERT_EQ(2, schema.size());+ ASSERT_EQ(4, schema.size());ASSERT_EQ(".*", schema[0].name);ASSERT_EQ("year", schema[1].name);+ ASSERT_EQ("kinds.CGXX", schema[2].name);+ ASSERT_EQ("kinds.ZBXX", schema[3].name);++ // filter on object key+ results = coll1->search("*", {}, "kinds.CGXX: 13", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());collectionManager.drop_collection("coll1");}@@ -1460,7 +1467,7 @@ TEST_F(CollectionAllFieldsTest, EmptyArrayShouldBeAcceptedAsFirstValueOfAutoFielcollectionManager.drop_collection("coll1");}-TEST_F(CollectionAllFieldsTest, SchemaUpdateShouldBeAtomicForAllFields) {+TEST_F(CollectionAllFieldsTest, DISABLED_SchemaUpdateShouldBeAtomicForAllFields) {// when a given field in a document is "bad", other fields should not be partially added to schemaCollection *coll1;@@ -1477,12 +1484,17 @@ TEST_F(CollectionAllFieldsTest, SchemaUpdateShouldBeAtomicForAllFields) {// insert a document with bad data for that key, but surrounded by "good" keys// this should NOT end up creating schema changesnlohmann::json doc;+ doc["int_2"] = 200;++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+doc["int_1"] = 100;doc["int_2"] = nlohmann::json::array();doc["int_2"].push_back(nlohmann::json::object());doc["int_3"] = 300;- auto add_op = coll1->add(doc.dump(), CREATE);+ add_op = coll1->add(doc.dump(), CREATE);ASSERT_FALSE(add_op.ok());auto f = coll1->get_fields();diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex b8faad9e..ca8fa26b 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -57,15 +57,15 @@ TEST_F(CollectionManagerTest, CollectionCreation) {collection1 = collectionManager2.get_collection("collection1").get();ASSERT_NE(nullptr, collection1);- std::unordered_map<std::string, field> schema = collection1->get_schema();+ tsl::htrie_map<char, field> schema = collection1->get_schema();std::vector<std::string> facet_fields_expected = {"cast"};ASSERT_EQ(0, collection1->get_collection_id());ASSERT_EQ(0, collection1->get_next_seq_id());ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());ASSERT_EQ(2, collection1->get_sort_fields().size());- ASSERT_EQ("points", collection1->get_sort_fields()[0].name);- ASSERT_EQ("location", collection1->get_sort_fields()[1].name);+ ASSERT_EQ("location", collection1->get_sort_fields()[0].name);+ ASSERT_EQ("points", collection1->get_sort_fields()[1].name);ASSERT_EQ(schema.size(), collection1->get_schema().size());ASSERT_EQ("points", collection1->get_default_sorting_field());ASSERT_EQ(false, schema.at("not_stored").index);@@ -287,7 +287,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {nlohmann::json results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(4, results["hits"].size());- std::unordered_map<std::string, field> schema = collection1->get_schema();+ tsl::htrie_map<char, field> schema = collection1->get_schema();// recreate collection manager to ensure that it restores the records from the disk backed storecollectionManager.dispose();@@ -312,8 +312,8 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(18, collection1->get_next_seq_id());ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());ASSERT_EQ(2, collection1->get_sort_fields().size());- ASSERT_EQ("points", collection1->get_sort_fields()[0].name);- ASSERT_EQ("location", collection1->get_sort_fields()[1].name);+ ASSERT_EQ("location", collection1->get_sort_fields()[0].name);+ ASSERT_EQ("points", collection1->get_sort_fields()[1].name);ASSERT_EQ(schema.size(), collection1->get_schema().size());ASSERT_EQ("points", collection1->get_default_sorting_field());@@ -446,7 +446,7 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {Option<nlohmann::json> add_op = coll1->add(doc_json, CREATE, "", DIRTY_VALUES::COERCE_OR_DROP);ASSERT_TRUE(add_op.ok());- std::unordered_map<std::string, field> schema = collection1->get_schema();+ tsl::htrie_map<char, field> schema = collection1->get_schema();// create a new collection manager to ensure that it restores the records from the disk backed storeCollectionManager & collectionManager2 = CollectionManager::get_instance();@@ -470,8 +470,8 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {ASSERT_EQ(7, restored_coll->get_num_documents());ASSERT_EQ(facet_fields_expected, restored_coll->get_facet_fields());ASSERT_EQ(3, restored_coll->get_sort_fields().size());- ASSERT_EQ("is_valid", restored_coll->get_sort_fields()[0].name);- ASSERT_EQ("average", restored_coll->get_sort_fields()[1].name);+ ASSERT_EQ("average", restored_coll->get_sort_fields()[0].name);+ ASSERT_EQ("is_valid", restored_coll->get_sort_fields()[1].name);ASSERT_EQ("max", restored_coll->get_sort_fields()[2].name);// ensures that the "id" field is not added to the schema@@ -488,11 +488,11 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {// all detected schema are optional fields, while defined schema is not- for(const auto& kv: restored_schema) {- if(kv.first == "max") {- ASSERT_FALSE(kv.second.optional);+ for(const auto& a_field: restored_schema) {+ if(a_field.name == "max") {+ ASSERT_FALSE(a_field.optional);} else {- ASSERT_TRUE(kv.second.optional);+ ASSERT_TRUE(a_field.optional);}}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppnew file mode 100644index 00000000..5703cfb3--- /dev/null+++ b/test/collection_nested_fields_test.cpp@@ -0,0 +1,523 @@+#include <gtest/gtest.h>+#include <string>+#include <vector>+#include <fstream>+#include <algorithm>+#include <collection_manager.h>+#include "collection.h"++class CollectionNestedFieldsTest : public ::testing::Test {+protected:+ Store* store;+ CollectionManager& collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;++ std::vector <std::string> query_fields;+ std::vector <sort_by> sort_fields;++ void setupCollection() {+ std::string state_dir_path = "/tmp/typesense_test/collection_nested";+ LOG(INFO) << "Truncating and creating: " << state_dir_path;+ system(("rm -rf " + state_dir_path + " && mkdir -p " + state_dir_path).c_str());++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);+ }++ virtual void SetUp() {+ setupCollection();+ }++ virtual void TearDown() {+ collectionManager.dispose();+ delete store;+ }+};++TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {+ auto json_str = R"({+ "company": {"name": "nike"},+ "employees": { "num": 1200 },+ "locations": [+ { "pincode": 100, "country": "USA",+ "address": { "street": "One Bowerman Drive", "city": "Beaverton", "products": ["shoes", "tshirts"] }+ },+ { "pincode": 200, "country": "Canada",+ "address": { "street": "175 Commerce Valley", "city": "Thornhill", "products": ["sneakers", "shoes"] }+ }+ ]}+ )";++ std::vector<field> nested_fields = {+ field("locations", field_types::OBJECT_ARRAY, false)+ };++ // array of objects+ std::vector<field> flattened_fields;+ nlohmann::json doc = nlohmann::json::parse(json_str);+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_EQ(5, flattened_fields.size());++ for(const auto& f: flattened_fields) {+ ASSERT_TRUE(f.is_array());+ }++ auto expected_json = R"(+ {+ ".flat": ["locations.address.city","locations.address.products","locations.address.street",+ "locations.country","locations.pincode"],+ "company":{"name":"nike"},+ "employees":{"num":1200},+ "locations":[+ {"address":{"city":"Beaverton","products":["shoes","tshirts"],+ "street":"One Bowerman Drive"},"country":"USA","pincode":100},++ {"address":{"city":"Thornhill","products":["sneakers","shoes"],+ "street":"175 Commerce Valley"},"country":"Canada","pincode":200}+ ],++ "locations.address.city":["Beaverton","Thornhill"],+ "locations.address.products":["shoes","tshirts","sneakers","shoes"],+ "locations.address.street":["One Bowerman Drive","175 Commerce Valley"],+ "locations.country":["USA","Canada"],+ "locations.pincode":[100,200]+ }+ )";++ ASSERT_EQ(doc.dump(), nlohmann::json::parse(expected_json).dump());++ // plain object+ flattened_fields.clear();+ doc = nlohmann::json::parse(json_str);+ nested_fields = {+ field("company", field_types::OBJECT, false)+ };++ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());++ expected_json = R"(+ {+ ".flat": ["company.name"],+ "company":{"name":"nike"},+ "company.name":"nike",+ "employees":{"num":1200},+ "company.name":"nike",+ "locations":[+ {"address":{"city":"Beaverton","products":["shoes","tshirts"],+ "street":"One Bowerman Drive"},"country":"USA","pincode":100},+ {"address":{"city":"Thornhill","products":["sneakers","shoes"],"street":"175 Commerce Valley"},+ "country":"Canada","pincode":200}+ ]+ }+ )";++ ASSERT_EQ(doc.dump(), nlohmann::json::parse(expected_json).dump());++ // plain object inside an array+ flattened_fields.clear();+ doc = nlohmann::json::parse(json_str);+ nested_fields = {+ field("locations.address", field_types::OBJECT, false)+ };++ ASSERT_FALSE(field::flatten_doc(doc, nested_fields, flattened_fields).ok()); // must be of type object_array++ nested_fields = {+ field("locations.address", field_types::OBJECT_ARRAY, false)+ };++ flattened_fields.clear();+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());++ expected_json = R"(+ {+ ".flat": ["locations.address.city", "locations.address.products", "locations.address.street"],+ "company":{"name":"nike"},+ "employees":{"num":1200},+ "locations":[+ {"address":{"city":"Beaverton","products":["shoes","tshirts"],+ "street":"One Bowerman Drive"},"country":"USA","pincode":100},+ {"address":{"city":"Thornhill","products":["sneakers","shoes"],"street":"175 Commerce Valley"},+ "country":"Canada","pincode":200}+ ],+ "locations.address.city":["Beaverton","Thornhill"],+ "locations.address.products":["shoes","tshirts","sneakers","shoes"],+ "locations.address.street":["One Bowerman Drive","175 Commerce Valley"]+ }+ )";++ ASSERT_EQ(doc.dump(), nlohmann::json::parse(expected_json).dump());++ // primitive inside nested object+ flattened_fields.clear();+ doc = nlohmann::json::parse(json_str);+ nested_fields = {+ field("company.name", field_types::STRING, false)+ };++ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());++ expected_json = R"(+ {+ ".flat": ["company.name"],+ "company":{"name":"nike"},+ "company.name":"nike",+ "employees":{"num":1200},+ "locations":[+ {"address":{"city":"Beaverton","products":["shoes","tshirts"],+ "street":"One Bowerman Drive"},"country":"USA","pincode":100},+ {"address":{"city":"Thornhill","products":["sneakers","shoes"],"street":"175 Commerce Valley"},+ "country":"Canada","pincode":200}+ ]+ }+ )";++ ASSERT_EQ(doc.dump(), nlohmann::json::parse(expected_json).dump());+}++TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {+ auto json_str = R"({+ "company": {"name": "nike"},+ "employees": {+ "num": 1200,+ "detail": {+ "num_tags": 2,+ "tags": ["plumber", "electrician"]+ },+ "details": [{+ "num_tags": 2,+ "tags": ["plumber", "electrician"]+ }]+ },+ "locations": [+ { "pincode": 100, "country": "USA",+ "address": { "street": "One Bowerman Drive", "city": "Beaverton", "products": ["shoes", "tshirts"] }+ },+ { "pincode": 200, "country": "Canada",+ "address": { "street": "175 Commerce Valley", "city": "Thornhill", "products": ["sneakers", "shoes"] }+ }+ ]}+ )";++ std::vector<field> nested_fields = {+ field("locations", field_types::OBJECT_ARRAY, false)+ };++ // array of objects+ std::vector<field> flattened_fields;+ nlohmann::json doc = nlohmann::json::parse(json_str);+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_EQ(5, flattened_fields.size());++ for(const auto& f: flattened_fields) {+ ASSERT_TRUE(f.is_array());+ ASSERT_TRUE(f.nested_array);+ }++ flattened_fields.clear();++ // test against whole object++ nested_fields = {+ field("employees", field_types::OBJECT, false)+ };++ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_EQ(5, flattened_fields.size());++ for(const auto& f: flattened_fields) {+ if(StringUtils::begins_with(f.name, "employees.details")) {+ ASSERT_TRUE(f.nested_array);+ } else {+ ASSERT_FALSE(f.nested_array);+ }+ }++ // test against deep paths+ flattened_fields.clear();+ nested_fields = {+ field("employees.details.num_tags", field_types::INT32_ARRAY, false),+ field("employees.details.tags", field_types::STRING_ARRAY, false),+ field("employees.detail.tags", field_types::STRING_ARRAY, false),+ };++ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_EQ(3, flattened_fields.size());++ ASSERT_EQ("employees.detail.tags",flattened_fields[0].name);+ ASSERT_FALSE(flattened_fields[0].nested_array);++ ASSERT_EQ("employees.details.num_tags",flattened_fields[1].name);+ ASSERT_TRUE(flattened_fields[1].nested_array);++ ASSERT_EQ("employees.details.tags",flattened_fields[2].name);+ ASSERT_TRUE(flattened_fields[2].nested_array);+}++TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {+ auto json_str = R"({+ "company": {"name": "nike"},+ "employees": { "num": 1200 }+ })";++ std::vector<field> nested_fields = {+ field("locations", field_types::OBJECT_ARRAY, false)+ };+ std::vector<field> flattened_fields;++ nlohmann::json doc = nlohmann::json::parse(json_str);+ auto flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);+ ASSERT_FALSE(flatten_op.ok());+ ASSERT_EQ("Field `locations` was not found or has an incorrect type.", flatten_op.error());++ nested_fields = {+ field("company", field_types::INT32, false)+ };++ flattened_fields.clear();+ flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);+ ASSERT_FALSE(flatten_op.ok());+ ASSERT_EQ("Field `company` was not found or has an incorrect type.", flatten_op.error());+}++TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {+ std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};++ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc = R"({+ "id": "0",+ "company": {"name": "Nike Inc."},+ "employees": {+ "num": 1200,+ "tags": ["senior plumber", "electrician"]+ },+ "locations": [+ { "pincode": 100, "country": "USA",+ "address": { "street": "One Bowerman Drive", "city": "Beaverton", "products": ["shoes", "tshirts"] }+ },+ { "pincode": 200, "country": "Canada",+ "address": { "street": "175 Commerce Valley", "city": "Thornhill", "products": ["sneakers", "shoes"] }+ }+ ]+ })"_json;++ auto add_op = coll1->add(doc.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());+ nlohmann::json create_res = add_op.get();+ ASSERT_EQ(doc.dump(), create_res.dump());++ // search both simply nested and deeply nested array-of-objects+ auto results = coll1->search("electrician commerce", {"employees", "locations"}, "", {}, sort_fields,+ {0}, 10, 1, FREQUENCY, {true}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(doc, results["hits"][0]["document"]);++ auto highlight_doc = R"({+ "employees":{+ "tags":[+ "senior plumber",+ "<mark>electrician</mark>"+ ]+ },+ "locations":[+ {+ "address":{+ "city":"Beaverton",+ "products":[+ "shoes",+ "tshirts"+ ],+ "street":"One Bowerman Drive"+ },+ "country":"USA"+ },+ {+ "address":{+ "city":"Thornhill",+ "products":[+ "sneakers",+ "shoes"+ ],+ "street":"175 <mark>Commerce</mark> Valley"+ },+ "country":"Canada"+ }+ ]+ })"_json;++ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());+ ASSERT_EQ(0, results["hits"][0]["highlights"].size());++ // search specific nested fields+ results = coll1->search("one shoe", {"locations.address.street", "employees.tags"}, "", {}, sort_fields,+ {0}, 10, 1, FREQUENCY, {true}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(doc, results["hits"][0]["document"]);++ highlight_doc = R"({+ "employees":{+ "tags":[+ "senior plumber",+ "electrician"+ ]+ },+ "locations":[+ {+ "address":{+ "street":"<mark>One</mark> Bowerman Drive"+ }+ },+ {+ "address":{+ "street":"<mark>One</mark> Bowerman Drive"+ }+ }+ ]+ })"_json;++ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());+ ASSERT_EQ(0, results["hits"][0]["highlights"].size());++ // try to search nested fields that don't exist+ auto res_op = coll1->search("one shoe", {"locations.address.str"}, "", {}, sort_fields,+ {0}, 10, 1, FREQUENCY, {true});+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `locations.address.str` in the schema.", res_op.error());++ res_op = coll1->search("one shoe", {"locations.address.foo"}, "", {}, sort_fields,+ {0}, 10, 1, FREQUENCY, {true});+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `locations.address.foo` in the schema.", res_op.error());++ res_op = coll1->search("one shoe", {"locations.foo.street"}, "", {}, sort_fields,+ {0}, 10, 1, FREQUENCY, {true});+ ASSERT_FALSE(res_op.ok());+ ASSERT_EQ("Could not find a field named `locations.foo.street` in the schema.", res_op.error());+}++TEST_F(CollectionNestedFieldsTest, IncludeExcludeFields) {+ auto doc_str = R"({+ "company": {"name": "Nike Inc."},+ "employees": {+ "num": 1200,+ "tags": ["senior plumber", "electrician"]+ },+ "employee": true,+ "locations": [+ { "pincode": 100, "country": "USA",+ "address": { "street": "One Bowerman Drive", "city": "Beaverton", "products": ["shoes", "tshirts"] }+ },+ { "pincode": 200, "country": "Canada",+ "address": { "street": "175 Commerce Valley", "city": "Thornhill", "products": ["sneakers", "shoes"] }+ }+ ],+ "one_obj_arr": [{"foo": "bar"}]+ })";++ auto doc = nlohmann::json::parse(doc_str);++ Collection::prune_doc(doc, tsl::htrie_set<char>(), {"one_obj_arr.foo"});+ ASSERT_EQ(0, doc.count("one_obj_arr"));++ // handle non-existing exclude field+ doc = nlohmann::json::parse(doc_str);+ Collection::prune_doc(doc, {"employees.num", "employees.tags"}, {"foobar"});+ ASSERT_EQ(1, doc.size());+ ASSERT_EQ(1, doc.count("employees"));+ ASSERT_EQ(2, doc["employees"].size());++ // select a specific field within nested array object+ doc = nlohmann::json::parse(doc_str);+ Collection::prune_doc(doc, {"locations.address.city"}, tsl::htrie_set<char>());+ ASSERT_EQ(R"({"locations":[{"address":{"city":"Beaverton"}},{"address":{"city":"Thornhill"}}]})", doc.dump());++ // select 2 fields within nested array object+ doc = nlohmann::json::parse(doc_str);+ Collection::prune_doc(doc, {"locations.address.city", "locations.address.products"}, tsl::htrie_set<char>());+ ASSERT_EQ(R"({"locations":[{"address":{"city":"Beaverton","products":["shoes","tshirts"]}},{"address":{"city":"Thornhill","products":["sneakers","shoes"]}}]})", doc.dump());++ // exclusion takes preference+ doc = nlohmann::json::parse(doc_str);+ Collection::prune_doc(doc, {"locations.address.city"}, {"locations.address.city"});+ ASSERT_EQ(R"({})", doc.dump());++ // include object, exclude sub-fields+ doc = nlohmann::json::parse(doc_str);+ Collection::prune_doc(doc, {"locations.address.city", "locations.address.products"}, {"locations.address.city"});+ ASSERT_EQ(R"({"locations":[{"address":{"products":["shoes","tshirts"]}},{"address":{"products":["sneakers","shoes"]}}]})", doc.dump());+}++TEST_F(CollectionNestedFieldsTest, GroupByOnNestedFieldsWithWildcardSchema) {+ std::vector<field> fields = {field(".*", field_types::AUTO, false, true),+ field("education.name", field_types::STRING_ARRAY, true, true),+ field("employee.num", field_types::INT32, true, true)};++ auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "employee": {"num": 5000},+ "education": [+ {"name": "X High School", "type": "school"},+ {"name": "Y University", "type": "undergraduate"}+ ]+ })"_json;++ auto doc2 = R"({+ "employee": {"num": 1000},+ "education": [+ {"name": "X High School", "type": "school"},+ {"name": "Z University", "type": "undergraduate"}+ ]+ })"_json;++ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());+ ASSERT_TRUE(coll1->add(doc2.dump(), CREATE).ok());++ // group on a field inside array of objects+ auto results = coll1->search("school", {"education"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10,+ spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,+ 5, "", 10, {}, {}, {"education.name"}, 2).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["grouped_hits"].size());++ ASSERT_EQ(1, results["grouped_hits"][0]["group_key"].size());+ ASSERT_EQ(2, results["grouped_hits"][0]["group_key"][0].size());+ ASSERT_EQ("X High School", results["grouped_hits"][0]["group_key"][0][0].get<std::string>());+ ASSERT_EQ("Z University", results["grouped_hits"][0]["group_key"][0][1].get<std::string>());+ ASSERT_EQ(1, results["grouped_hits"][0]["hits"].size());+ ASSERT_EQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());++ ASSERT_EQ(1, results["grouped_hits"][1]["group_key"].size());+ ASSERT_EQ(2, results["grouped_hits"][1]["group_key"][0].size());+ ASSERT_EQ("X High School", results["grouped_hits"][1]["group_key"][0][0].get<std::string>());+ ASSERT_EQ("Y University", results["grouped_hits"][1]["group_key"][0][1].get<std::string>());+ ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());+ ASSERT_EQ("0", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());++ // group on plain nested field+ results = coll1->search("school", {"education"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10,+ spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,+ 5, "", 10, {}, {}, {"employee.num"}, 2).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["grouped_hits"].size());++ ASSERT_EQ(1, results["grouped_hits"][0]["group_key"].size());+ ASSERT_EQ(1, results["grouped_hits"][0]["group_key"][0].size());+ ASSERT_EQ(1000, results["grouped_hits"][0]["group_key"][0].get<size_t>());+ ASSERT_EQ(1, results["grouped_hits"][0]["hits"].size());+ ASSERT_EQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());++ ASSERT_EQ(1, results["grouped_hits"][1]["group_key"].size());+ ASSERT_EQ(1, results["grouped_hits"][1]["group_key"][0].size());+ ASSERT_EQ(5000, results["grouped_hits"][1]["group_key"][0].get<size_t>());+ ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());+ ASSERT_EQ("0", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());+}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex fedb6a0f..a7e8a2a2 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2153,37 +2153,42 @@ TEST_F(CollectionTest, SearchLargeTextField) {TEST_F(CollectionTest, PruneFieldsFromDocument) {nlohmann::json document = get_prune_doc();- Collection::prune_document(document, {"one", "two"}, spp::sparse_hash_set<std::string>());+ Collection::prune_doc(document, {"one", "two"}, tsl::htrie_set<char>());ASSERT_EQ(2, document.size());ASSERT_EQ(1, document["one"]);ASSERT_EQ(2, document["two"]);// exclude takes precedencedocument = get_prune_doc();- Collection::prune_document(document, {"one"}, {"one"});+ Collection::prune_doc(document, {"one"}, {"one"});ASSERT_EQ(0, document.size());// when no inclusion is specified, should return all fields not mentioned by exclusion listdocument = get_prune_doc();- Collection::prune_document(document, spp::sparse_hash_set<std::string>(), {"three"});+ Collection::prune_doc(document, tsl::htrie_set<char>(), tsl::htrie_set<char>({"three"}), "");ASSERT_EQ(3, document.size());ASSERT_EQ(1, document["one"]);ASSERT_EQ(2, document["two"]);ASSERT_EQ(4, document["four"]);document = get_prune_doc();- Collection::prune_document(document, spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>());+ Collection::prune_doc(document, tsl::htrie_set<char>(), tsl::htrie_set<char>(), "");ASSERT_EQ(4, document.size());// when included field does not existdocument = get_prune_doc();- Collection::prune_document(document, {"notfound"}, spp::sparse_hash_set<std::string>());+ Collection::prune_doc(document, {"notfound"}, tsl::htrie_set<char>(), "");ASSERT_EQ(0, document.size());// when excluded field does not existdocument = get_prune_doc();- Collection::prune_document(document, spp::sparse_hash_set<std::string>(), {"notfound"});+ Collection::prune_doc(document, tsl::htrie_set<char>(), {"notfound"}, "");ASSERT_EQ(4, document.size());++ // included set is prefix of allowed fields+ document = get_prune_doc();+ Collection::prune_doc(document, {"ones"}, tsl::htrie_set<char>(), "");+ ASSERT_EQ(0, document.size());}TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {diff --git a/test/index_test.cpp b/test/index_test.cppindex 69c338cf..d788d5f5 100644--- a/test/index_test.cpp+++ b/test/index_test.cpp@@ -4,7 +4,7 @@#include <s2/s2loop.h>TEST(IndexTest, ScrubReindexDoc) {- std::unordered_map<std::string, field> search_schema;+ tsl::htrie_map<char, field> search_schema;search_schema.emplace("title", field("title", field_types::STRING, false));search_schema.emplace("points", field("points", field_types::INT32, false));search_schema.emplace("cast", field("cast", field_types::STRING_ARRAY, false));