Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactored the facet implementation and as well as the query interface.
Kishore Nallan7af95e7f2295
diff --git a/TODO.md b/TODO.mdindex e53459c7..2d6f9044 100644--- a/TODO.md+++ b/TODO.md@@ -21,35 +21,37 @@- ~~storage key prefix should include collection name~~- ~~Index and search on multi-valued field~~- ~~range search for art_int~~-- Proper score field for ranking tokens-- Support nested fields via "."- ~~Restore records as well on restart (like for meta)~~- ~~drop collection should remove all records from the store~~-- Pagination parameter-- UTF-8 support for fuzzy search- ~~Multi-key binary search during scoring~~- ~~Assumption that all tokens match for scoring is no longer true~~-- Handle searching for non-existing fields gracefully-- Intersection without unpacking- ~~Filters~~- ~~Facets~~+- Prevent string copy during indexing+- Schema validation during insertion+- clean special chars before indexing+- Proper score field for ranking tokens+- Minimum results should be a variable instead of blindly going with max_results+- Pagination parameter- Iterator- Highlight+- UTF-8 support for fuzzy search+- Handle searching for non-existing fields gracefully+- test for same match score but different primary, secondary attr+- only last token should be prefix searched+- Intersection without unpacking+- Support nested fields via "."- Support search operators like +, - etc.- Prefix-search strings should not be null terminated- string_utils::tokenize should not have max length-- only last token should be prefix searched- art float search-- Minimum results should be a variable instead of blindly going with max_results- Benchmark with -ffast-math- Space sensitivity- Use bitmap index instead of compressed array for doc list- Throw errors when schema is broken- Primary_rank_scores and secondary_rank_scores hashmaps should be combined- Proper logging-- clean special chars before indexing- d-ary heap?-- test for same match score but different primary, secondary attr**API**diff --git a/include/collection.h b/include/collection.hindex 0749782e..05f7fa64 100644--- a/include/collection.h+++ b/include/collection.h@@ -19,17 +19,19 @@ private:// Auto incrementing record ID used internally for indexing - not exposed to the clientuint32_t next_seq_id;- spp::sparse_hash_map<std::string, field> schema;+ spp::sparse_hash_map<std::string, field> search_schema;++ spp::sparse_hash_map<std::string, field> facet_schema;std::vector<std::string> rank_fields;Store* store;- spp::sparse_hash_map<std::string, art_tree*> index_map;+ spp::sparse_hash_map<std::string, art_tree*> search_index;- spp::sparse_hash_map<uint32_t, int64_t> primary_rank_scores;+ spp::sparse_hash_map<std::string, art_tree*> facet_index;- spp::sparse_hash_map<uint32_t, int64_t> secondary_rank_scores;+ spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> rank_index;std::string get_doc_id_key(std::string doc_id);@@ -46,17 +48,21 @@ private:void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);- void search(uint32_t *filter_ids, size_t filter_ids_length, std::vector<facet> &facets, std::string &query,- const std::string &field, const int num_typos, const size_t num_results, Topster<100> &topster,- size_t &num_found, const token_ordering token_order = FREQUENCY, const bool prefix = false);+ void search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,+ std::vector<facet> & facets, const std::vector<std::string> & rank_fields, const int num_typos,+ const size_t num_results, Topster<100> &topster, size_t & num_found,+ const token_ordering token_order = FREQUENCY, const bool prefix = false);void search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,- int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,+ const std::vector<std::string> & rank_fields, int & token_rank,+ std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,size_t & total_results, size_t & num_found, const size_t & max_results);- void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const;+ void index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id,+ const bool verbatim) const;- void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t, uint32_t seq_id) const;+ void index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,+ uint32_t seq_id, const bool verbatim) const;void index_int32_field(const int32_t value, uint32_t score, art_tree *t, uint32_t seq_id) const;@@ -73,7 +79,8 @@ public:Collection() = delete;Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,- const std::vector<field> & search_fields, const std::vector<std::string> rank_fields);+ const std::vector<field> & search_fields, const std::vector<field> & facet_fields,+ const std::vector<std::string> & rank_fields);~Collection();@@ -89,20 +96,24 @@ public:uint32_t doc_id_to_seq_id(std::string doc_id);+ std::vector<std::string> get_facet_fields();+std::vector<std::string> get_rank_fields();spp::sparse_hash_map<std::string, field> get_schema();std::string add(std::string json_str);- nlohmann::json search(std::string query, const std::vector<std::string> fields, const std::string &simple_filter_query,- std::vector<facet> & facets, const int num_typos, const size_t num_results,- const token_ordering token_order = FREQUENCY, const bool prefix = false);+ nlohmann::json search(std::string query, const std::vector<std::string> search_fields,+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,+ const std::vector<std::string> & rank_fields, const int num_typos,+ const size_t num_results, const token_ordering token_order = FREQUENCY, const bool prefix = false);void remove(std::string id);- void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,- const uint32_t *result_ids, const size_t result_size) const;+ void score_results(const std::vector<std::string> & rank_fields, const int & token_rank, Topster<100> &topster,+ const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,+ const size_t result_size) const;void index_in_memory(const nlohmann::json &document, uint32_t seq_id);diff --git a/include/collection_manager.h b/include/collection_manager.hindex 675cdf30..2191ca8e 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -21,6 +21,7 @@ private:static constexpr const char* COLLECTION_NAME_KEY = "name";static constexpr const char* COLLECTION_ID_KEY = "id";static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "search_fields";+ static constexpr const char* COLLECTION_FACET_FIELDS_KEY = "facet_fields";static constexpr const char* COLLECTION_RANK_FIELDS_KEY = "rank_fields";CollectionManager();@@ -39,7 +40,8 @@ public:void init(Store *store);Collection* create_collection(std::string name, const std::vector<field> & search_fields,- const std::vector<std::string> & rank_fields);+ const std::vector<field> & facet_fields,+ const std::vector<std::string> & rank_fields);Collection* get_collection(std::string collection_name);diff --git a/src/collection.cpp b/src/collection.cppindex 8492206e..5d9db218 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -8,23 +8,46 @@#include <art.h>Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,- const std::vector<field> &search_fields, const std::vector<std::string> rank_fields):+ const std::vector<field> &search_fields, const std::vector<field> & facet_fields,+ const std::vector<std::string> & rank_fields):name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store), rank_fields(rank_fields) {for(const field& field: search_fields) {art_tree *t = new art_tree;art_tree_init(t);- index_map.emplace(field.name, t);- schema.emplace(field.name, field);+ search_index.emplace(field.name, t);+ search_schema.emplace(field.name, field);+ }++ for(const field& field: facet_fields) {+ art_tree *t = new art_tree;+ art_tree_init(t);+ facet_index.emplace(field.name, t);+ facet_schema.emplace(field.name, field);+ }++ for(const std::string & rank_field: rank_fields) {+ spp::sparse_hash_map<uint32_t, int64_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, int64_t>();+ rank_index.emplace(rank_field, doc_to_score);}}Collection::~Collection() {- for(std::pair<std::string, field> name_field: schema) {- art_tree *t = index_map.at(name_field.first);+ for(std::pair<std::string, field> name_field: search_schema) {+ art_tree *t = search_index.at(name_field.first);art_tree_destroy(t);t = nullptr;}++ for(std::pair<std::string, field> name_field: facet_schema) {+ art_tree *t = facet_index.at(name_field.first);+ art_tree_destroy(t);+ t = nullptr;+ }++ for(std::pair<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> name_map: rank_index) {+ delete name_map.second;+ }}uint32_t Collection::get_next_seq_id() {@@ -50,18 +73,20 @@ std::string Collection::add(std::string json_str) {}void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id) {- for(const std::pair<std::string, field> & field_pair: schema) {- const std::string & field_name = field_pair.first;- art_tree *t = index_map.at(field_name);+ // FIXME: field might not exist in the document or field type might be invalid - need to validate!- uint32_t points = 0;- if(document.count("points") != 0) {- points = document["points"];- }+ uint32_t points = 0;+ if(document.count("points") != 0) {+ points = document["points"];+ }++ for(const std::pair<std::string, field> & field_pair: search_schema) {+ const std::string & field_name = field_pair.first;+ art_tree *t = search_index.at(field_name);if(field_pair.second.type == field_types::STRING) {const std::string & text = document[field_name];- index_string_field(text, points, t, seq_id);+ index_string_field(text, points, t, seq_id, false);} else if(field_pair.second.type == field_types::INT32) {uint32_t value = document[field_name];index_int32_field(value, points, t, seq_id);@@ -70,7 +95,7 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_idindex_int64_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {std::vector<std::string> strings = document[field_name];- index_string_array_field(strings, points, t, seq_id);+ index_string_array_field(strings, points, t, seq_id, false);} else if(field_pair.second.type == field_types::INT32_ARRAY) {std::vector<int32_t> values = document[field_name];index_int32_array_field(values, points, t, seq_id);@@ -80,12 +105,24 @@ void Collection::index_in_memory(const nlohmann::json &document, uint32_t seq_id}}- if(rank_fields.size() > 0 && document.count(rank_fields[0])) {- primary_rank_scores[seq_id] = document[rank_fields[0]].get<int64_t>();+ for(const std::pair<std::string, field> & field_pair: facet_schema) {+ const std::string & field_name = field_pair.first;+ art_tree *t = facet_index.at(field_name);+ if(field_pair.second.type == field_types::STRING) {+ const std::string & text = document[field_name];+ index_string_field(text, points, t, seq_id, true);+ } else if(field_pair.second.type == field_types::STRING_ARRAY) {+ std::vector<std::string> strings = document[field_name];+ index_string_array_field(strings, points, t, seq_id, true);+ }}- if(rank_fields.size() > 1 && document.count(rank_fields[1])) {- secondary_rank_scores[seq_id] = document[rank_fields[1]].get<int64_t>();+ for(const std::string & rank_field: rank_fields) {+ if(rank_index.count(rank_field) > 0) {+ spp::sparse_hash_map<uint32_t, int64_t> *doc_to_score = rank_index.at(rank_field);+ doc_to_score->emplace(seq_id, document[rank_fields[0]].get<int64_t>());+ }+ // FIXME: handle else (return error)}}@@ -135,15 +172,21 @@ void Collection::index_int64_field(const int64_t value, uint32_t score, art_treeart_insert(t, key, KEY_LEN, &art_doc, num_hits);}-void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t, uint32_t seq_id) const {+void Collection::index_string_field(const std::string & text, const uint32_t score, art_tree *t,+ uint32_t seq_id, const bool verbatim) const {std::vector<std::string> tokens;- StringUtils::tokenize(text, tokens, " ", true);-std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- for(uint32_t i=0; i<tokens.size(); i++) {- auto token = tokens[i];- transform(token.begin(), token.end(), token.begin(), tolower);- token_to_offsets[token].push_back(i);++ if(verbatim) {+ tokens.push_back(text);+ token_to_offsets[text].push_back(0);+ } else {+ StringUtils::tokenize(text, tokens, " ", true);+ for(uint32_t i=0; i<tokens.size(); i++) {+ auto token = tokens[i];+ transform(token.begin(), token.end(), token.begin(), tolower);+ token_to_offsets[token].push_back(i);+ }}for(auto & kv: token_to_offsets) {@@ -175,9 +218,9 @@ void Collection::index_string_field(const std::string & text, const uint32_t sco}void Collection::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,- uint32_t seq_id) const {+ uint32_t seq_id, const bool verbatim) const {for(const std::string & str: strings) {- index_string_field(str, score, t, seq_id);+ index_string_field(str, score, t, seq_id, verbatim);}}@@ -198,11 +241,11 @@ void Collection::index_int64_array_field(const std::vector<int64_t> & values, covoid Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size) {for(auto & a_facet: facets) {// assumed that facet fields have already been validated upstream- const field & facet_field = schema.at(a_facet.field_name);+ const field & facet_field = facet_schema.at(a_facet.field_name);// loop through the field, get all keys and intersect those ids with result ids- if(index_map.count(facet_field.name) != 0) {- art_tree *t = index_map.at(facet_field.name);+ if(facet_index.count(facet_field.name) != 0) {+ art_tree *t = facet_index.at(facet_field.name);std::vector<art_leaf *> leaves;art_topk_iter(t->root, MAX_SCORE, 10, leaves);@@ -226,9 +269,9 @@ void Collection::do_facets(std::vector<facet> & facets, uint32_t* result_ids, si}void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, std::vector<facet> & facets,- int & token_rank, std::vector<std::vector<art_leaf*>> & token_leaves,- Topster<100> & topster, size_t & total_results, size_t & num_found,- const size_t & max_results) {+ const std::vector<std::string> & rank_fields, int & token_rank,+ std::vector<std::vector<art_leaf*>> & token_leaves, Topster<100> & topster,+ size_t & total_results, size_t & num_found, const size_t & max_results) {const size_t combination_limit = 10;auto product = []( long long a, std::vector<art_leaf*>& b ) { return a*b.size(); };long long int N = std::accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product);@@ -261,7 +304,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengtdo_facets(facets, filtered_result_ids, filtered_results_size);// go through each matching document id and calculate match score- score_results(topster, token_rank, query_suggestion, filtered_result_ids, filtered_results_size);+ score_results(rank_fields, token_rank, topster, query_suggestion, filtered_result_ids, filtered_results_size);num_found += filtered_results_size;delete[] filtered_result_ids;@@ -269,7 +312,7 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt} else {do_facets(facets, result_ids, result_size);- score_results(topster, token_rank, query_suggestion, result_ids, result_size);+ score_results(rank_fields, token_rank, topster, query_suggestion, result_ids, result_size);num_found += result_size;delete[] result_ids;}@@ -317,11 +360,11 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::}const std::string & field_name = expression_parts[0];- if(schema.count(field_name) == 0) {+ if(search_schema.count(field_name) == 0) {return Option<>(400, "Could not find a filter field named `" + field_name + "` in the schema.");}- field _field = schema.at(field_name);+ field _field = search_schema.at(field_name);const std::string & raw_value = expression_parts[1];filter f;@@ -381,9 +424,9 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::// process the filters firstfor(const filter & a_filter: filters) {- if(index_map.count(a_filter.field_name) != 0) {- art_tree* t = index_map.at(a_filter.field_name);- field f = schema.at(a_filter.field_name);+ if(search_index.count(a_filter.field_name) != 0) {+ art_tree* t = search_index.at(a_filter.field_name);+ field f = search_schema.at(a_filter.field_name);std::vector<const art_leaf*> leaves;if(f.integer()) {@@ -425,22 +468,41 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::return Option<>(filter_ids_length);}-nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields,- const std::string & simple_filter_query, std::vector<facet> & facets,- const int num_typos, const size_t num_results, const token_ordering token_order,- const bool prefix) {+nlohmann::json Collection::search(std::string query, const std::vector<std::string> search_fields,+ const std::string & simple_filter_query, const std::vector<std::string> & facet_fields,+ const std::vector<std::string> & rank_fields, const int num_typos,+ const size_t num_results, const token_ordering token_order, const bool prefix) {size_t num_found = 0;nlohmann::json result = nlohmann::json::object();+ std::vector<facet> facets;++ // validate search fields+ for(const std::string & field_name: search_fields) {+ if(search_schema.count(field_name) == 0) {+ result["error"] = "Could not find a search field named `" + field_name + "` in the schema.";+ return result;+ }++ field search_field = search_schema.at(field_name);+ if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {+ result["error"] = "Search field `" + field_name + "` should be a string or a string array.";+ return result;+ }+ }// validate facet fields- for(const facet & a_facet: facets) {- if(schema.count(a_facet.field_name) == 0) {- result["error"] = "Could not find a facet field named `" + a_facet.field_name + "` in the schema.";+ for(const std::string & field_name: facet_fields) {+ if(facet_schema.count(field_name) == 0) {+ result["error"] = "Could not find a facet field named `" + field_name + "` in the schema.";return result;}- field facet_field = schema.at(a_facet.field_name);- if(facet_field.type != field_types::STRING && facet_field.type != field_types::STRING_ARRAY) {- result["error"] = "Facet field `" + a_facet.field_name + "` should be a string or a string array.";+ facets.push_back(facet(field_name));+ }++ // validate rank fields+ for(const std::string & field_name: rank_fields) {+ if(rank_index.count(field_name) == 0) {+ result["error"] = "Could not find a rank field named `" + field_name + "` in the schema.";return result;}}@@ -459,18 +521,18 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striauto begin = std::chrono::high_resolution_clock::now();std::vector<std::pair<int, Topster<100>::KV>> field_order_kvs;- for(int i = 0; i < fields.size(); i++) {+ for(int i = 0; i < search_fields.size(); i++) {Topster<100> topster;- const std::string & field = fields[i];+ const std::string & field = search_fields[i];// proceed to query search only when no filters are provided or when filtering produces resultsif(simple_filter_query.size() == 0 || filter_ids_length > 0) {- search(filter_ids, filter_ids_length, facets, query, field, num_typos, num_results, topster, num_found,- token_order, prefix);+ search_field(query, field, filter_ids, filter_ids_length, facets, rank_fields, num_typos, num_results,+ topster, num_found, token_order, prefix);topster.sort();}for(auto t = 0; t < topster.size && t < num_results; t++) {- field_order_kvs.push_back(std::make_pair(fields.size() - i, topster.getKV(t)));+ field_order_kvs.push_back(std::make_pair(search_fields.size() - i, topster.getKV(t)));}}@@ -497,14 +559,22 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::striresult["found"] = num_found;- result["facets"] = nlohmann::json::array();+ result["facet_counts"] = nlohmann::json::array();// populate facetsfor(const facet & a_facet: facets) {nlohmann::json facet_result = nlohmann::json::object();facet_result["field_name"] = a_facet.field_name;- facet_result["counts"] = a_facet.result_map;- result["facets"].push_back(facet_result);+ facet_result["counts"] = nlohmann::json::array();++ for(auto kv: a_facet.result_map) {+ nlohmann::json facet_value_count = nlohmann::json::object();+ facet_value_count["value"] = kv.first;+ facet_value_count["count"] = kv.second;+ facet_result["counts"].push_back(facet_value_count);+ }++ result["facet_counts"].push_back(facet_result);}long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();@@ -522,9 +592,10 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vector<facet> &facets, std::string &query,- const std::string &field, const int num_typos, const size_t num_results, Topster<100> &topster,- size_t &num_found, const token_ordering token_order, const bool prefix) {+void Collection::search_field(std::string & query, const std::string & field, uint32_t *filter_ids, size_t filter_ids_length,+ std::vector<facet> & facets, const std::vector<std::string> & rank_fields, const int num_typos,+ const size_t num_results, Topster<100> &topster, size_t & num_found,+ const token_ordering token_order, const bool prefix) {std::vector<std::string> tokens;StringUtils::tokenize(query, tokens, " ", true);@@ -590,7 +661,7 @@ void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vecstd::cout << "\n";}- art_fuzzy_search(index_map.at(field), (const unsigned char *) token.c_str(), token_len,+ art_fuzzy_search(search_index.at(field), (const unsigned char *) token.c_str(), token_len,costs[token_index], costs[token_index], 3, token_order, prefix, leaves);if(!leaves.empty()) {@@ -627,7 +698,7 @@ void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vecif(token_leaves.size() != 0 && token_leaves.size() == tokens.size()) {// If all tokens were found, go ahead and search for candidates with what we have so far- search_candidates(filter_ids, filter_ids_length, facets, token_rank, token_leaves, topster,+ search_candidates(filter_ids, filter_ids_length, facets, rank_fields, token_rank, token_leaves, topster,total_results, num_found, max_results);if (total_results >= max_results) {@@ -661,8 +732,8 @@ void Collection::search(uint32_t *filter_ids, size_t filter_ids_length, std::vec}}- return search(filter_ids, filter_ids_length, facets, truncated_query, field, num_typos, num_results, topster,- num_found, token_order, prefix);+ return search_field(truncated_query, field, filter_ids, filter_ids_length, facets, rank_fields, num_typos,+ num_results, topster, num_found, token_order, prefix);}}@@ -677,9 +748,9 @@ void Collection::log_leaves(const int cost, const std::string &token, const std:}}-void Collection::score_results(Topster<100> & topster, const int & token_rank,- const std::vector<art_leaf *> &query_suggestion, const uint32_t *result_ids,- const size_t result_size) const {+void Collection::score_results(const std::vector<std::string> & rank_fields, const int & token_rank,+ Topster<100> & topster, const std::vector<art_leaf *> &query_suggestion,+ const uint32_t *result_ids, const size_t result_size) const {const int max_token_rank = 250;spp::sparse_hash_map<art_leaf*, uint32_t*> leaf_to_indices;@@ -693,6 +764,18 @@ void Collection::score_results(Topster<100> & topster, const int & token_rank,}}+ spp::sparse_hash_map<uint32_t, int64_t> * primary_rank_scores = nullptr;+ spp::sparse_hash_map<uint32_t, int64_t> * secondary_rank_scores = nullptr;++ if(rank_fields.size() > 0) {+ // assumed that rank field exists in the index - checked earlier in the chain+ primary_rank_scores = rank_index.at(rank_fields[0]);+ }++ if(rank_fields.size() > 1) {+ secondary_rank_scores = rank_index.at(rank_fields[1]);+ }+for(auto i=0; i<result_size; i++) {uint32_t seq_id = result_ids[i];std::vector<std::vector<uint16_t>> token_positions;@@ -733,8 +816,9 @@ void Collection::score_results(Topster<100> & topster, const int & token_rank,((uint64_t)(mscore.words_present) << 8) +(MAX_SEARCH_TOKENS - mscore.distance);- int64_t primary_rank_score = primary_rank_scores.count(seq_id) > 0 ? primary_rank_scores.at(seq_id) : 0;- int64_t secondary_rank_score = secondary_rank_scores.count(seq_id) > 0 ? secondary_rank_scores.at(seq_id) : 0;+ int64_t primary_rank_score = primary_rank_scores->count(seq_id) > 0 ? primary_rank_scores->at(seq_id) : 0;+ int64_t secondary_rank_score = (secondary_rank_scores && secondary_rank_scores->count(seq_id) > 0) ?+ secondary_rank_scores->at(seq_id) : 0;topster.add(seq_id, match_score, primary_rank_score, secondary_rank_score);/*std::cout << "token_rank_score: " << token_rank_score << ", match_score: "<< match_score << ", primary_rank_score: " << primary_rank_score << ", seq_id: " << seq_id << std::endl;*/@@ -818,7 +902,7 @@ void Collection::remove(std::string id) {const unsigned char *key = (const unsigned char *) token.c_str();int key_len = (int) (token.length() + 1);- art_leaf* leaf = (art_leaf *) art_search(index_map.at("title"), key, key_len);+ art_leaf* leaf = (art_leaf *) art_search(search_index.at("title"), key, key_len);if(leaf != NULL) {uint32_t seq_id_values[1] = {seq_id};@@ -841,7 +925,7 @@ void Collection::remove(std::string id) {std::cout << "----" << std::endl;*/if(leaf->values->ids.getLength() == 0) {- art_delete(index_map.at("title"), key, key_len);+ art_delete(search_index.at("title"), key, key_len);}}}@@ -880,12 +964,21 @@ uint32_t Collection::doc_id_to_seq_id(std::string doc_id) {return seq_id;}+std::vector<std::string> Collection::get_facet_fields() {+ std::vector<std::string> facet_fields_copy;+ for(auto it = facet_schema.begin(); it != facet_schema.end(); ++it) {+ facet_fields_copy.push_back(it->first);+ }++ return facet_fields_copy;+}+std::vector<std::string> Collection::get_rank_fields() {return rank_fields;}spp::sparse_hash_map<std::string, field> Collection::get_schema() {- return schema;+ return search_schema;};std::string Collection::get_meta_key(std::string collection_name) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 475b81b4..96b2cafd 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -28,11 +28,18 @@ void CollectionManager::init(Store *store) {std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();std::vector<field> search_fields;- nlohmann::json fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];+ nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];- for (nlohmann::json::iterator it = fields_map.begin(); it != fields_map.end(); ++it) {+ for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});}++ std::vector<field> facet_fields;+ nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];++ for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {+ facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});+ }std::string collection_next_seq_id_str;store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);@@ -46,6 +53,7 @@ void CollectionManager::init(Store *store) {collection_next_seq_id,store,search_fields,+ facet_fields,collection_rank_fields);// Fetch records from the store and re-create memory index@@ -70,7 +78,8 @@ void CollectionManager::init(Store *store) {}Collection* CollectionManager::create_collection(std::string name, const std::vector<field> & search_fields,- const std::vector<std::string> & rank_fields) {+ const std::vector<field> & facet_fields,+ const std::vector<std::string> & rank_fields) {if(store->contains(Collection::get_meta_key(name))) {return nullptr;}@@ -85,12 +94,21 @@ Collection* CollectionManager::create_collection(std::string name, const std::vesearch_fields_json.push_back(field_val);}+ nlohmann::json facet_fields_json = nlohmann::json::array();;+ for(const field& facet_field: facet_fields) {+ nlohmann::json field_val;+ field_val[fields::name] = facet_field.name;+ field_val[fields::type] = facet_field.type;+ facet_fields_json.push_back(field_val);+ }+collection_meta[COLLECTION_NAME_KEY] = name;collection_meta[COLLECTION_ID_KEY] = next_collection_id;collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = search_fields_json;+ collection_meta[COLLECTION_FACET_FIELDS_KEY] = facet_fields_json;collection_meta[COLLECTION_RANK_FIELDS_KEY] = rank_fields;- Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, rank_fields);+ Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, facet_fields, rank_fields);store->insert(Collection::get_meta_key(name), collection_meta.dump());store->insert(Collection::get_next_seq_id_key(name), std::to_string(0));diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 70bcadbe..e1fc5c83 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -10,7 +10,8 @@ protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();Collection *collection1;- std::vector<field> fields;+ std::vector<field> search_fields;+ std::vector<field> facet_fields;std::vector<std::string> rank_fields;void setupCollection() {@@ -21,9 +22,11 @@ protected:store = new Store(state_dir_path);collectionManager.init(store);- fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};+ search_fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};+ facet_fields = {field("starring", field_types::STRING)};rank_fields = {"points"};- collection1 = collectionManager.create_collection("collection1", fields, rank_fields);++ collection1 = collectionManager.create_collection("collection1", search_fields, facet_fields, rank_fields);}virtual void SetUp() {@@ -47,9 +50,9 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {infile.close();std::vector<std::string> search_fields = {"starring", "title"};- std::vector<facet> facets;+ std::vector<std::string> facets;- nlohmann::json results = collection1->search("thomas", search_fields, "", facets, 0, 10, FREQUENCY, false);+ nlohmann::json results = collection1->search("thomas", search_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());spp::sparse_hash_map<std::string, field> schema = collection1->get_schema();@@ -61,12 +64,15 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {collection1 = collectionManager2.get_collection("collection1");ASSERT_NE(nullptr, collection1);+ std::vector<std::string> facet_fields_expected = {facet_fields[0].name};+ASSERT_EQ(0, collection1->get_collection_id());ASSERT_EQ(18, collection1->get_next_seq_id());+ ASSERT_EQ(facet_fields_expected, collection1->get_facet_fields());ASSERT_EQ(rank_fields, collection1->get_rank_fields());ASSERT_EQ(schema.size(), collection1->get_schema().size());- results = collection1->search("thomas", search_fields, "", facets, 0, 10, FREQUENCY, false);+ results = collection1->search("thomas", search_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 23409ab6..c485c460 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -8,9 +8,11 @@class CollectionTest : public ::testing::Test {protected:Collection *collection;- std::vector<std::string> search_fields;+ std::vector<std::string> query_fields;Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::vector<std::string> rank_fields;+ std::vector<field> facet_fields;void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/collection";@@ -21,13 +23,15 @@ protected:collectionManager.init(store);std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");- std::vector<field> fields = {field("title", field_types::STRING)};- std::vector<std::string> rank_fields = {"points"};- search_fields = {"title"};+ std::vector<field> search_fields = {field("title", field_types::STRING)};++ query_fields = {"title"};+ facet_fields = { };+ rank_fields = {"points"};collection = collectionManager.get_collection("collection");if(collection == nullptr) {- collection = collectionManager.create_collection("collection", fields, rank_fields);+ collection = collectionManager.create_collection("collection", search_fields, facet_fields, rank_fields);}std::string json_line;@@ -54,8 +58,8 @@ protected:};TEST_F(CollectionTest, ExactSearchShouldBeStable) {- std::vector<facet> facets;- nlohmann::json results = collection->search("the", search_fields, "", facets, 0, 10);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("the", query_fields, "", facets, rank_fields, 0, 10);ASSERT_EQ(7, results["hits"].size());ASSERT_EQ(7, results["found"].get<int>());@@ -71,8 +75,8 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {}TEST_F(CollectionTest, ExactPhraseSearch) {- std::vector<facet> facets;- nlohmann::json results = collection->search("rocket launch", search_fields, "", facets, 0, 10);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, rank_fields, 0, 10);ASSERT_EQ(5, results["hits"].size());/*@@ -94,7 +98,7 @@ TEST_F(CollectionTest, ExactPhraseSearch) {}// Check pagination- results = collection->search("rocket launch", search_fields, "", facets, 0, 3);+ results = collection->search("rocket launch", query_fields, "", facets, rank_fields, 0, 3);ASSERT_EQ(3, results["hits"].size());for(size_t i = 0; i < 3; i++) {nlohmann::json result = results["hits"].at(i);@@ -106,8 +110,8 @@ TEST_F(CollectionTest, ExactPhraseSearch) {TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {// Tokens that are not found in the index should be skipped- std::vector<facet> facets;- nlohmann::json results = collection->search("DoesNotExist from", search_fields, "", facets, 0, 10);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("DoesNotExist from", query_fields, "", facets, rank_fields, 0, 10);ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"2", "17"};@@ -120,7 +124,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}// with non-zero cost- results = collection->search("DoesNotExist from", search_fields, "", facets, 1, 10);+ results = collection->search("DoesNotExist from", query_fields, "", facets, rank_fields, 1, 10);ASSERT_EQ(2, results["hits"].size());for(size_t i = 0; i < results["hits"].size(); i++) {@@ -131,7 +135,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}// with 2 indexed words- results = collection->search("from DoesNotExist insTruments", search_fields, "", facets, 1, 10);+ results = collection->search("from DoesNotExist insTruments", query_fields, "", facets, rank_fields, 1, 10);ASSERT_EQ(2, results["hits"].size());ids = {"2", "17"};@@ -143,17 +147,17 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {}results.clear();- results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, "", facets, 0, 10);+ results = collection->search("DoesNotExist1 DoesNotExist2", query_fields, "", facets, rank_fields, 0, 10);ASSERT_EQ(0, results["hits"].size());results.clear();- results = collection->search("DoesNotExist1 DoesNotExist2", search_fields, "", facets, 2, 10);+ results = collection->search("DoesNotExist1 DoesNotExist2", query_fields, "", facets, rank_fields, 2, 10);ASSERT_EQ(0, results["hits"].size());}TEST_F(CollectionTest, PartialPhraseSearch) {- std::vector<facet> facets;- nlohmann::json results = collection->search("rocket research", search_fields, "", facets, 0, 10);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("rocket research", query_fields, "", facets, rank_fields, 0, 10);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"1", "8", "16", "17"};@@ -167,8 +171,8 @@ TEST_F(CollectionTest, PartialPhraseSearch) {}TEST_F(CollectionTest, QueryWithTypo) {- std::vector<facet> facets;- nlohmann::json results = collection->search("kind biologcal", search_fields, "", facets, 2, 3);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("kind biologcal", query_fields, "", facets, rank_fields, 2, 3);ASSERT_EQ(3, results["hits"].size());std::vector<std::string> ids = {"19", "20", "21"};@@ -181,7 +185,7 @@ TEST_F(CollectionTest, QueryWithTypo) {}results.clear();- results = collection->search("fer thx", search_fields, "", facets, 1, 3);+ results = collection->search("fer thx", query_fields, "", facets, rank_fields, 1, 3);ids = {"1", "10", "13"};ASSERT_EQ(3, results["hits"].size());@@ -195,8 +199,8 @@ TEST_F(CollectionTest, QueryWithTypo) {}TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {- std::vector<facet> facets;- nlohmann::json results = collection->search("loox", search_fields, "", facets, 1, 2, MAX_SCORE, false);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 2, MAX_SCORE, false);ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"22", "23"};@@ -207,7 +211,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = collection->search("loox", search_fields, "", facets, 1, 3, FREQUENCY, false);+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 3, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"3", "12", "24"};@@ -219,19 +223,19 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {}// Check pagination- results = collection->search("loox", search_fields, "", facets, 1, 1, FREQUENCY, false);+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 1, FREQUENCY, false);ASSERT_EQ(3, results["found"].get<int>());ASSERT_EQ(1, results["hits"].size());std::string solo_id = results["hits"].at(0)["id"];ASSERT_STREQ("3", solo_id.c_str());- results = collection->search("loox", search_fields, "", facets, 1, 2, FREQUENCY, false);+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 2, FREQUENCY, false);ASSERT_EQ(3, results["found"].get<int>());ASSERT_EQ(2, results["hits"].size());// Check total ordering- results = collection->search("loox", search_fields, "", facets, 1, 10, FREQUENCY, false);+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 10, FREQUENCY, false);ASSERT_EQ(5, results["hits"].size());ids = {"3", "12", "24", "22", "23"};@@ -242,7 +246,7 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = collection->search("loox", search_fields, "", facets, 1, 10, MAX_SCORE, false);+ results = collection->search("loox", query_fields, "", facets, rank_fields, 1, 10, MAX_SCORE, false);ASSERT_EQ(5, results["hits"].size());ids = {"22", "23", "3", "12", "24"};@@ -256,8 +260,8 @@ TEST_F(CollectionTest, TypoTokenRankedByScoreAndFrequency) {TEST_F(CollectionTest, TextContainingAnActualTypo) {// A line contains "ISX" but not "what" - need to ensure that correction to "ISS what" happens- std::vector<facet> facets;- nlohmann::json results = collection->search("ISX what", search_fields, "", facets, 1, 4, FREQUENCY, false);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("ISX what", query_fields, "", facets, rank_fields, 1, 4, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"19", "6", "21", "8"};@@ -270,7 +274,7 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {}// Record containing exact token match should appear first- results = collection->search("ISX", search_fields, "", facets, 1, 10, FREQUENCY, false);+ results = collection->search("ISX", query_fields, "", facets, rank_fields, 1, 10, FREQUENCY, false);ASSERT_EQ(8, results["hits"].size());ids = {"20", "19", "6", "3", "21", "4", "10", "8"};@@ -284,8 +288,8 @@ TEST_F(CollectionTest, TextContainingAnActualTypo) {}TEST_F(CollectionTest, PrefixSearching) {- std::vector<facet> facets;- nlohmann::json results = collection->search("ex", search_fields, "", facets, 0, 10, FREQUENCY, true);+ std::vector<std::string> facets;+ nlohmann::json results = collection->search("ex", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, true);ASSERT_EQ(2, results["hits"].size());std::vector<std::string> ids = {"12", "6"};@@ -296,7 +300,7 @@ TEST_F(CollectionTest, PrefixSearching) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = collection->search("ex", search_fields, "", facets, 0, 10, MAX_SCORE, true);+ results = collection->search("ex", query_fields, "", facets, rank_fields, 0, 10, MAX_SCORE, true);ASSERT_EQ(2, results["hits"].size());ids = {"6", "12"};@@ -318,7 +322,7 @@ TEST_F(CollectionTest, MultipleFields) {coll_mul_fields = collectionManager.get_collection("coll_mul_fields");if(coll_mul_fields == nullptr) {- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, rank_fields);+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, rank_fields);}std::string json_line;@@ -329,9 +333,9 @@ TEST_F(CollectionTest, MultipleFields) {infile.close();- search_fields = {"title", "starring"};- std::vector<facet> facets;- nlohmann::json results = coll_mul_fields->search("Will", search_fields, "", facets, 0, 10, FREQUENCY, false);+ query_fields = {"title", "starring"};+ std::vector<std::string> facets;+ nlohmann::json results = coll_mul_fields->search("Will", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"3", "2", "1", "0"};@@ -345,8 +349,8 @@ TEST_F(CollectionTest, MultipleFields) {// when "starring" takes higher priority than "title"- search_fields = {"starring", "title"};- results = coll_mul_fields->search("thomas", search_fields, "", facets, 0, 10, FREQUENCY, false);+ query_fields = {"starring", "title"};+ results = coll_mul_fields->search("thomas", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());ids = {"15", "14", "12", "13"};@@ -358,12 +362,12 @@ TEST_F(CollectionTest, MultipleFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- search_fields = {"starring", "title", "cast"};- results = coll_mul_fields->search("ben affleck", search_fields, "", facets, 0, 10, FREQUENCY, false);+ query_fields = {"starring", "title", "cast"};+ results = coll_mul_fields->search("ben affleck", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(1, results["hits"].size());- search_fields = {"cast"};- results = coll_mul_fields->search("chris", search_fields, "", facets, 0, 10, FREQUENCY, false);+ query_fields = {"cast"};+ results = coll_mul_fields->search("chris", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"6", "1", "7"};@@ -374,8 +378,8 @@ TEST_F(CollectionTest, MultipleFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- search_fields = {"cast"};- results = coll_mul_fields->search("chris pine", search_fields, "", facets, 0, 10, FREQUENCY, false);+ query_fields = {"cast"};+ results = coll_mul_fields->search("chris pine", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"7", "6", "1"};@@ -398,7 +402,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);}std::string json_line;@@ -410,9 +414,9 @@ TEST_F(CollectionTest, FilterOnNumericFields) {infile.close();// Plain search with no filters - results should be sorted by rank fields- search_fields = {"name"};- std::vector<facet> facets;- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "", facets, 0, 10, FREQUENCY, false);+ query_fields = {"name"};+ std::vector<std::string> facets;+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(5, results["hits"].size());std::vector<std::string> ids = {"3", "1", "4", "0", "2"};@@ -425,7 +429,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {}// Searching on an int32 field- results = coll_array_fields->search("Jeremy", search_fields, "age:>24", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "age:>24", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"3", "1", "4"};@@ -437,14 +441,14 @@ TEST_F(CollectionTest, FilterOnNumericFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = coll_array_fields->search("Jeremy", search_fields, "age:>=24", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "age:>=24", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());- results = coll_array_fields->search("Jeremy", search_fields, "age:24", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "age:24", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(1, results["hits"].size());// Searching a number against an int32 array field- results = coll_array_fields->search("Jeremy", search_fields, "years:>2002", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "years:>2002", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"1", "0", "2"};@@ -455,7 +459,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = coll_array_fields->search("Jeremy", search_fields, "years:<1989", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "years:<1989", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(1, results["hits"].size());ids = {"3"};@@ -467,7 +471,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {}// multiple filters- results = coll_array_fields->search("Jeremy", search_fields, "years:<2005 && years:>1987", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "years:<2005 && years:>1987", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(1, results["hits"].size());ids = {"4"};@@ -479,7 +483,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {}// multiple search values (works like SQL's IN operator) against a single int field- results = coll_array_fields->search("Jeremy", search_fields, "age:[21, 24, 63]", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "age:[21, 24, 63]", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(3, results["hits"].size());ids = {"3", "0", "2"};@@ -491,7 +495,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {}// multiple search values against an int32 array field - also use extra padding between symbols- results = coll_array_fields->search("Jeremy", search_fields, "years : [ 2015, 1985 , 1999]", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "years : [ 2015, 1985 , 1999]", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());ids = {"3", "1", "4", "0"};@@ -503,7 +507,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {}// searching on an int64 array field - also ensure that padded space causes no issues- results = coll_array_fields->search("Jeremy", search_fields, "timestamps : > 475205222", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps : > 475205222", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());ids = {"1", "4", "0", "2"};@@ -516,7 +520,7 @@ TEST_F(CollectionTest, FilterOnNumericFields) {}// when filters don't match any record, no results should be returned- results = coll_array_fields->search("Jeremy", search_fields, "timestamps:<1", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps:<1", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());collectionManager.drop_collection("coll_array_fields");@@ -533,7 +537,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);}std::string json_line;@@ -544,9 +548,9 @@ TEST_F(CollectionTest, FilterOnTextFields) {infile.close();- search_fields = {"name"};- std::vector<facet> facets;- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "tags: gold", facets, 0, 10, FREQUENCY, false);+ query_fields = {"name"};+ std::vector<std::string> facets;+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "tags: gold", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());std::vector<std::string> ids = {"1", "4", "0", "2"};@@ -558,7 +562,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- results = coll_array_fields->search("Jeremy", search_fields, "tags : bronze", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "tags : bronze", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(2, results["hits"].size());ids = {"4", "2"};@@ -571,7 +575,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {}// search with a list of tags, also testing extra padding of space- results = coll_array_fields->search("Jeremy", search_fields, "tags: [bronze, silver]", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "tags: [bronze, silver]", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(4, results["hits"].size());ids = {"3", "4", "0", "2"};@@ -584,7 +588,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {}// should be exact matches (no normalization or fuzzy searching should happen)- results = coll_array_fields->search("Jeremy", search_fields, "tags: BRONZE", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "tags: BRONZE", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());collectionManager.drop_collection("coll_array_fields");@@ -603,7 +607,7 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);}std::string json_line;@@ -614,32 +618,34 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {infile.close();- search_fields = {"name"};- std::vector<facet> facets;+ query_fields = {"name"};+ std::vector<std::string> facets;// when filter field does not exist in the schema- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "tagzz: gold", facets, 0, 10, FREQUENCY, false);+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "tagzz: gold", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());// searching using a string for a numeric field- results = coll_array_fields->search("Jeremy", search_fields, "age: abcdef", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "age: abcdef", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());// searching using a string for a numeric array field- results = coll_array_fields->search("Jeremy", search_fields, "timestamps: abcdef", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps: abcdef", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());// malformed k:v syntax- results = coll_array_fields->search("Jeremy", search_fields, "timestamps abcdef", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "timestamps abcdef", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());// just empty spaces- results = coll_array_fields->search("Jeremy", search_fields, " ", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, " ", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());// wrapping number with quotes- results = coll_array_fields->search("Jeremy", search_fields, "age: '21'", facets, 0, 10, FREQUENCY, false);+ results = coll_array_fields->search("Jeremy", query_fields, "age: '21'", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(0, results["hits"].size());++ collectionManager.drop_collection("coll_array_fields");}TEST_F(CollectionTest, FacetCounts) {@@ -650,11 +656,12 @@ TEST_F(CollectionTest, FacetCounts) {field("years", field_types::INT32_ARRAY),field("timestamps", field_types::INT64_ARRAY),field("tags", field_types::STRING_ARRAY)};+ facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};std::vector<std::string> rank_fields = {"age"};coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, rank_fields);+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);}std::string json_line;@@ -665,45 +672,113 @@ TEST_F(CollectionTest, FacetCounts) {infile.close();- search_fields = {"name"};- std::vector<facet> facets = {facet("tags")};+ query_fields = {"name"};+ std::vector<std::string> facets = {"tags"};// single facet with no filters- nlohmann::json results = coll_array_fields->search("Jeremy", search_fields, "", facets, 0, 10, FREQUENCY, false);+ nlohmann::json results = coll_array_fields->search("Jeremy", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);ASSERT_EQ(5, results["hits"].size());- ASSERT_EQ(1, results["facets"].size());- ASSERT_EQ(2, results["facets"][0].size());- ASSERT_EQ("tags", results["facets"][0]["field_name"]);+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0].size());+ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);++ ASSERT_EQ("gold", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ(4, (int) results["facet_counts"][0]["counts"][1]["count"]);++ ASSERT_EQ("silver", results["facet_counts"][0]["counts"][2]["value"]);+ ASSERT_EQ(3, (int) results["facet_counts"][0]["counts"][2]["count"]);- ASSERT_EQ(4, (int) results["facets"][0]["counts"]["gold"]);- ASSERT_EQ(3, (int) results["facets"][0]["counts"]["silver"]);- ASSERT_EQ(2, (int) results["facets"][0]["counts"]["bronze"]);+ ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][0]["count"]);// 2 facets, 1 text filter with no filtersfacets.clear();- facets.push_back(facet("tags"));- facets.push_back(facet("name"));- results = coll_array_fields->search("Jeremy", search_fields, "", facets, 0, 10, FREQUENCY, false);+ facets.push_back("tags");+ facets.push_back("name");+ results = coll_array_fields->search("Jeremy", query_fields, "", facets, rank_fields, 0, 10, FREQUENCY, false);+ASSERT_EQ(5, results["hits"].size());- ASSERT_EQ(2, results["facets"].size());+ ASSERT_EQ(2, results["facet_counts"].size());- ASSERT_EQ("tags", results["facets"][0]["field_name"]);- ASSERT_EQ("name", results["facets"][1]["field_name"]);+ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);+ ASSERT_EQ("name", results["facet_counts"][1]["field_name"]);- // text is tokenized and standardized- ASSERT_EQ(5, (int) results["facets"][1]["counts"]["howard"]);- ASSERT_EQ(5, (int) results["facets"][1]["counts"]["jeremy"]);+ // facet value must one that's stored, not indexed (i.e. no tokenization/standardization)+ ASSERT_EQ("Jeremy Howard", results["facet_counts"][1]["counts"][0]["value"]);+ ASSERT_EQ(5, (int) results["facet_counts"][1]["counts"][0]["count"]);// facet with filtersfacets.clear();- facets.push_back(facet("tags"));- results = coll_array_fields->search("Jeremy", search_fields, "age: >24", facets, 0, 10, FREQUENCY, false);+ facets.push_back("tags");+ results = coll_array_fields->search("Jeremy", query_fields, "age: >24", facets, rank_fields, 0, 10, FREQUENCY, false);+ASSERT_EQ(3, results["hits"].size());- ASSERT_EQ(1, results["facets"].size());+ ASSERT_EQ(1, results["facet_counts"].size());++ ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);+ ASSERT_EQ(1, (int) results["facet_counts"][0]["counts"][0]["count"]);+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][1]["count"]);+ ASSERT_EQ(2, (int) results["facet_counts"][0]["counts"][2]["count"]);++ ASSERT_EQ("bronze", results["facet_counts"][0]["counts"][0]["value"]);+ ASSERT_EQ("gold", results["facet_counts"][0]["counts"][1]["value"]);+ ASSERT_EQ("silver", results["facet_counts"][0]["counts"][2]["value"]);++ collectionManager.drop_collection("coll_array_fields");+}++TEST_F(CollectionTest, SearchingWithMissingFields) {+ // return error without crashing when searching for fields that do not conform to the schema+ Collection *coll_array_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");+ std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),+ field("years", field_types::INT32_ARRAY),+ field("timestamps", field_types::INT64_ARRAY),+ field("tags", field_types::STRING_ARRAY)};+ facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};+ std::vector<std::string> rank_fields = {"age"};++ coll_array_fields = collectionManager.get_collection("coll_array_fields");+ if(coll_array_fields == nullptr) {+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, rank_fields);+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_array_fields->add(json_line);+ }++ infile.close();++ // when a query field mentioned in schema does not exist+ std::vector<std::string> facets;+ std::vector<std::string> query_fields_not_found = {"titlez"};++ nlohmann::json res = coll_array_fields->search("the", query_fields_not_found, "", facets, rank_fields, 0, 10);+ ASSERT_EQ(0, res["hits"].size());+ ASSERT_STREQ("Could not find a search field named `titlez` in the schema.",res["error"].get<std::string>().c_str());- ASSERT_EQ("tags", results["facets"][0]["field_name"]);- ASSERT_EQ(2, (int) results["facets"][0]["counts"]["gold"]);- ASSERT_EQ(2, (int) results["facets"][0]["counts"]["silver"]);- ASSERT_EQ(1, (int) results["facets"][0]["counts"]["bronze"]);+ // when a query field is an integer field+ res = coll_array_fields->search("the", {"age"}, "", facets, rank_fields, 0, 10);+ ASSERT_EQ(0, res["hits"].size());+ ASSERT_STREQ("Search field `age` should be a string or a string array.", res["error"].get<std::string>().c_str());++ // when a facet field is not defined in the schema+ res = coll_array_fields->search("the", {"name"}, "", {"timestamps"}, rank_fields, 0, 10);+ ASSERT_EQ(0, res["hits"].size());+ ASSERT_STREQ("Could not find a facet field named `timestamps` in the schema.", res["error"].get<std::string>().c_str());++ // when a rank field is not defined in the schema+ res = coll_array_fields->search("the", {"name"}, "", {}, {"timestamps"}, 0, 10);+ ASSERT_EQ(0, res["hits"].size());+ ASSERT_STREQ("Could not find a rank field named `timestamps` in the schema.", res["error"].get<std::string>().c_str());++ res = coll_array_fields->search("the", {"name"}, "", {}, {"_rank"}, 0, 10);+ ASSERT_EQ(0, res["hits"].size());+ ASSERT_STREQ("Could not find a rank field named `_rank` in the schema.", res["error"].get<std::string>().c_str());++ collectionManager.drop_collection("coll_array_fields");}\ No newline at end of file