Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Combine search, facet and sort fields into a single "fields".
Kishore Nallan0e4517d9019c
diff --git a/TODO.md b/TODO.mdindex 1a483ad9..01240c98 100644--- a/TODO.md+++ b/TODO.md@@ -67,6 +67,11 @@- ~~test for string filter comparison: title < "foo"~~- ~~Test for sorted_array::indexOf when length is 0~~- ~~Test for pagination~~+- ~~search_fields, sort_fields and facet fields should be combined~~+- ~~facet fields should be indexed verbatim~~+- change "search_by" to "query_by"+- during index_in_memory() validations should be front loaded+- NOT operator support- test for token ranking on float field- test for float int field deletion during doc deletion- Test for snippetsdiff --git a/include/collection.h b/include/collection.hindex b6bf155e..e499ef5c 100644--- a/include/collection.h+++ b/include/collection.h@@ -123,8 +123,7 @@ public:Collection() = delete;Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,- const std::vector<field> & search_fields, const std::vector<field> & facet_fields,- const std::vector<field> & sort_fields, const std::string token_ranking_field);+ const std::vector<field> & fields, const std::string & token_ranking_field);~Collection();diff --git a/include/collection_manager.h b/include/collection_manager.hindex 479c1a0f..dada354f 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -22,9 +22,7 @@ private:static constexpr const char* COLLECTION_NAME_KEY = "name";static constexpr const char* COLLECTION_ID_KEY = "id";- static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "search_fields";- static constexpr const char* COLLECTION_FACET_FIELDS_KEY = "facet_fields";- static constexpr const char* COLLECTION_SORT_FIELDS_KEY = "sort_fields";+ static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields";static constexpr const char* COLLECTION_TOKEN_ORDERING_FIELD_KEY = "token_ranking_field";std::string auth_key;@@ -53,10 +51,8 @@ public:bool auth_key_matches(std::string auth_key_sent);- Option<Collection*> create_collection(std::string name, const std::vector<field> & search_fields,- const std::vector<field> & facet_fields,- const std::vector<field> & sort_fields,- const std::string & token_ranking_field = "");+ Option<Collection*> create_collection(std::string name, const std::vector<field> & fields,+ const std::string & token_ranking_field = "");Collection* get_collection(const std::string & collection_name);diff --git a/include/field.h b/include/field.hindex 4408c400..f983e0df 100644--- a/include/field.h+++ b/include/field.h@@ -19,28 +19,42 @@ namespace field_types {namespace fields {static const std::string name = "name";static const std::string type = "type";+ static const std::string facet = "facet";}struct field {std::string name;std::string type;+ bool facet;- field(const std::string & name, const std::string & type): name(name), type(type) {+ field(const std::string & name, const std::string & type, const bool & facet): name(name), type(type), facet(facet) {}- bool is_integer() {+ bool is_single_integer() const {+ return (type == field_types::INT32 || type == field_types::INT64);+ }++ bool is_single_float() const {+ return (type == field_types::FLOAT);+ }++ bool is_integer() const {return (type == field_types::INT32 || type == field_types::INT32_ARRAY ||type == field_types::INT64 || type == field_types::INT64_ARRAY);}- bool is_float() {+ bool is_float() const {return (type == field_types::FLOAT || type == field_types::FLOAT_ARRAY);}- bool is_string() {+ bool is_string() const {return (type == field_types::STRING || type == field_types::STRING_ARRAY);}++ bool is_facet() const {+ return facet;+ }};struct filter {diff --git a/src/api.cpp b/src/api.cppindex 46b6ef54..93619a40 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -40,12 +40,8 @@ void post_create_collection(http_req & req, http_res & res) {return res.send_400("Parameter `name` is required.");}- if(req_json.count("search_fields") == 0) {- return res.send_400("Parameter `search_fields` is required.");- }-- if(req_json.count("sort_fields") == 0) {- return res.send_400("Parameter `sort_fields` is required.");+ if(req_json.count("fields") == 0) {+ return res.send_400("Parameter `fields` is required.");}if(collectionManager.get_collection(req_json["name"]) != nullptr) {@@ -54,69 +50,29 @@ void post_create_collection(http_req & req, http_res & res) {// field specific validation- std::vector<field> search_fields;+ std::vector<field> fields;- if(!req_json["search_fields"].is_array() || req_json["search_fields"].size() == 0) {- return res.send_400("Wrong format for `search_fields`. It should be an array like: "+ if(!req_json["fields"].is_array() || req_json["fields"].size() == 0) {+ return res.send_400("Wrong format for `fields`. It should be an array like: ""[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");}- for(const nlohmann::json & search_field_json: req_json["search_fields"]) {- if(!search_field_json.is_object() ||- search_field_json.count(fields::name) == 0 || search_field_json.count(fields::type) == 0 ||- !search_field_json.at(fields::name).is_string() || !search_field_json.at(fields::type).is_string()) {+ for(nlohmann::json & field_json: req_json["fields"]) {+ if(!field_json.is_object() ||+ field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 ||+ !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) {- return res.send_400("Wrong format for `search_fields`. It should be an array like: "+ return res.send_400("Wrong format for `fields`. It should be an array like: ""[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");}- search_fields.push_back(field(search_field_json["name"], search_field_json["type"]));- }-- std::vector<field> facet_fields;-- if(req_json.count("facet_fields") != 0) {- if(!req_json["facet_fields"].is_array()) {- return res.send_400("Wrong format for `facet_fields`. It should be an array like: "- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");- }-- for(const nlohmann::json & facet_field_json: req_json["facet_fields"]) {- if(!facet_field_json.is_object() ||- facet_field_json.count(fields::name) == 0 || facet_field_json.count(fields::type) == 0 ||- !facet_field_json.at(fields::name).is_string() || !facet_field_json.at(fields::type).is_string()) {-- return res.send_400("Wrong format for `facet_fields`. It should be an array like: "- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");- }-- facet_fields.push_back(field(facet_field_json["name"], facet_field_json["type"]));- }- }-- std::vector<field> sort_fields;-- if(!req_json["sort_fields"].is_array() || req_json["sort_fields"].size() == 0) {- return res.send_400("Wrong format for `sort_fields`. It should be an array like: "- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");- }-- for(const nlohmann::json & sort_field_json: req_json["sort_fields"]) {- if(!sort_field_json.is_object() ||- sort_field_json.count(fields::name) == 0 || sort_field_json.count(fields::type) == 0 ||- !sort_field_json.at(fields::name).is_string() ||- !sort_field_json.at(fields::type).is_string()) {-- return res.send_400("Wrong format for `sort_fields`. It should be an array like: "- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");- }-- const std::string & sort_field_type = sort_field_json["type"].get<std::string>();- if(sort_field_type != field_types::INT32 && sort_field_type != field_types::INT64) {- return res.send_400("Sort field `" + sort_field_json["name"].get<std::string>() + "` must be an integer.");+ if(field_json.count("facet") == 0) {+ field_json["facet"] = false;}- sort_fields.push_back(field(sort_field_json["name"], sort_field_json["type"]));+ fields.push_back(+ field(field_json["name"], field_json["type"], field_json["facet"])+ );}const char* PREFIX_RANKING_FIELD = "prefix_ranking_field";@@ -124,13 +80,14 @@ void post_create_collection(http_req & req, http_res & res) {if(req_json.count(PREFIX_RANKING_FIELD) != 0) {if(!req_json[PREFIX_RANKING_FIELD].is_string()) {- return res.send_400(std::string("`") + PREFIX_RANKING_FIELD + "` should be the name of an unsigned integer field.");+ return res.send_400(std::string("`") + PREFIX_RANKING_FIELD ++ "` should be a string. It should be the name of an unsigned integer field.");}token_ranking_field = req_json[PREFIX_RANKING_FIELD].get<std::string>();}- collectionManager.create_collection(req_json["name"], search_fields, facet_fields, sort_fields, token_ranking_field);+ collectionManager.create_collection(req_json["name"], fields, token_ranking_field);res.send_201(req.body);}diff --git a/src/collection.cpp b/src/collection.cppindex 318b12c9..0c1789b0 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -8,28 +8,27 @@#include <art.h>Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,- const std::vector<field> &search_fields, const std::vector<field> & facet_fields,- const std::vector<field> & sort_fields, const std::string token_ranking_field):+ const std::vector<field> &fields, const std::string & token_ranking_field):name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),token_ranking_field(token_ranking_field) {- for(const field& field: search_fields) {+ for(const field& field: fields) {art_tree *t = new art_tree;art_tree_init(t);search_index.emplace(field.name, t);search_schema.emplace(field.name, field);- }- for(const field& field: facet_fields) {- facet_value fvalue;- facet_index.emplace(field.name, fvalue);- facet_schema.emplace(field.name, field);- }+ if(field.is_facet()) {+ facet_value fvalue;+ facet_index.emplace(field.name, fvalue);+ facet_schema.emplace(field.name, field);+ }- for(const field & sort_field: sort_fields) {- spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();- sort_index.emplace(sort_field.name, doc_to_score);- sort_schema.emplace(sort_field.name, sort_field);+ if(field.is_single_integer() || field.is_single_float()) {+ spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();+ sort_index.emplace(field.name, doc_to_score);+ sort_schema.emplace(field.name, field);+ }}num_documents = 0;@@ -123,7 +122,7 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinconst std::string & field_name = field_pair.first;if(document.count(field_name) == 0) {- return Option<>(400, "Field `" + field_name + "` has been declared as a search field in the schema, "+ return Option<>(400, "Field `" + field_name + "` has been declared in the schema, ""but is not found in the document.");}@@ -131,80 +130,92 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinif(field_pair.second.type == field_types::STRING) {if(!document[field_name].is_string()) {- return Option<>(400, "Search field `" + field_name + "` must be a string.");+ return Option<>(400, "Field `" + field_name + "` must be a string.");}const std::string & text = document[field_name];- index_string_field(text, points, t, seq_id, false);+ index_string_field(text, points, t, seq_id, field_pair.second.is_facet());} else if(field_pair.second.type == field_types::INT32) {if(!document[field_name].is_number_integer()) {- return Option<>(400, "Search field `" + field_name + "` must be an int32.");+ return Option<>(400, "Field `" + field_name + "` must be an int32.");}if(document[field_name].get<int64_t>() > INT32_MAX) {- return Option<>(400, "Search field `" + field_name + "` exceeds maximum value of int32.");+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");}uint32_t value = document[field_name];index_int32_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::INT64) {if(!document[field_name].is_number_integer()) {- return Option<>(400, "Search field `" + field_name + "` must be an int64.");+ return Option<>(400, "Field `" + field_name + "` must be an int64.");}uint64_t value = document[field_name];index_int64_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::FLOAT) {if(!document[field_name].is_number_float()) {- return Option<>(400, "Search field `" + field_name + "` must be a float.");+ return Option<>(400, "Field `" + field_name + "` must be a float.");}float value = document[field_name];index_float_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {if(!document[field_name].is_array()) {- return Option<>(400, "Search field `" + field_name + "` must be a string array.");+ return Option<>(400, "Field `" + field_name + "` must be a string array.");}if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {- return Option<>(400, "Search field `" + field_name + "` must be a string array.");+ return Option<>(400, "Field `" + field_name + "` must be a string array.");}std::vector<std::string> strings = document[field_name];- index_string_array_field(strings, points, t, seq_id, false);+ index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());} else if(field_pair.second.type == field_types::INT32_ARRAY) {if(!document[field_name].is_array()) {- return Option<>(400, "Search field `" + field_name + "` must be an int32 array.");+ return Option<>(400, "Field `" + field_name + "` must be an int32 array.");}if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {- return Option<>(400, "Search field `" + field_name + "` must be an int32 array.");+ return Option<>(400, "Field `" + field_name + "` must be an int32 array.");}std::vector<int32_t> values = document[field_name];index_int32_array_field(values, points, t, seq_id);} else if(field_pair.second.type == field_types::INT64_ARRAY) {if(!document[field_name].is_array()) {- return Option<>(400, "Search field `" + field_name + "` must be an int64 array.");+ return Option<>(400, "Field `" + field_name + "` must be an int64 array.");}if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {- return Option<>(400, "Search field `" + field_name + "` must be an int64 array.");+ return Option<>(400, "Field `" + field_name + "` must be an int64 array.");}std::vector<int64_t> values = document[field_name];index_int64_array_field(values, points, t, seq_id);} else if(field_pair.second.type == field_types::FLOAT_ARRAY) {if(!document[field_name].is_array()) {- return Option<>(400, "Search field `" + field_name + "` must be a float array.");+ return Option<>(400, "Field `" + field_name + "` must be a float array.");}if(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) {- return Option<>(400, "Search field `" + field_name + "` must be a float array.");+ return Option<>(400, "Field `" + field_name + "` must be a float array.");}std::vector<float> values = document[field_name];index_float_array_field(values, points, t, seq_id);}++ // add numerical values automatically into sort index+ if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||+ field_pair.second.type == field_types::FLOAT) {+ spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);++ if(document[field_pair.first].is_number_integer()) {+ doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());+ } else if(document[field_pair.first].is_number_float()) {+ doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());+ }+ }}for(const std::pair<std::string, field> & field_pair: facet_schema) {@@ -233,27 +244,8 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uinconst std::vector<std::string> & values = document[field_name];fvalue.index_values(seq_id, values);- }- }-- for(const std::pair<std::string, field> & field_pair: sort_schema) {- const field & sort_field = field_pair.second;-- if(document.count(sort_field.name) == 0) {- return Option<>(400, "Field `" + sort_field.name + "` has been declared as a sort field in the schema, "- "but is not found in the document.");- }-- if(!document[sort_field.name].is_number()) {- return Option<>(400, "Sort field `" + sort_field.name + "` must be a number.");- }-- spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(sort_field.name);-- if(document[sort_field.name].is_number_integer()) {- doc_to_score->emplace(seq_id, document[sort_field.name].get<int64_t>());} else {- doc_to_score->emplace(seq_id, document[sort_field.name].get<float>());+ return Option<>(400, "Facet field `" + field_name + "` must be a string or a string[].");}}@@ -598,6 +590,10 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::f = {field_name, {filter_value}, op_comparator.get()};}} else if(_field.is_string()) {+ if(!_field.facet) {+ return Option<>(400, "Field `" + _field.name + "` is not a faceted field - only a field marked as a "+ "facet can be filtered on.");+ }if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {std::vector<std::string> filter_values;StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");@@ -678,13 +674,18 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// validate search fieldsfor(const std::string & field_name: search_fields) {if(search_schema.count(field_name) == 0) {- std::string error = "Could not find a search field named `" + field_name + "` in the schema.";+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";return Option<nlohmann::json>(400, error);}field search_field = search_schema.at(field_name);if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {- std::string error = "Search field `" + field_name + "` should be a string or a string array.";+ std::string error = "Field `" + field_name + "` should be a string or a string array.";+ return Option<nlohmann::json>(400, error);+ }++ if(search_field.facet) {+ std::string error = "Field `" + field_name + "` is a faceted field - it cannot be used as a query field.";return Option<nlohmann::json>(400, error);}}@@ -704,7 +705,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sfor(const sort_by & _sort_field: sort_fields) {if(sort_index.count(_sort_field.name) == 0) {- std::string error = "Could not find a sort field named `" + _sort_field.name + "` in the schema.";+ std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema.";return Option<nlohmann::json>(400, error);}@@ -712,7 +713,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sStringUtils::toupper(sort_order);if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {- std::string error = "Order for sort field` " + _sort_field.name + "` should be either ASC or DESC.";+ std::string error = "Order for field` " + _sort_field.name + "` should be either ASC or DESC.";return Option<nlohmann::json>(400, error);}@@ -1110,7 +1111,7 @@ void Collection::score_results(const std::vector<sort_by> & sort_fields, const i// initialize primary_rank_factorfield sort_field = sort_schema.at(sort_fields[0].name);- if(sort_field.is_integer()) {+ if(sort_field.is_single_integer()) {primary_rank_factor = ((int64_t) 1);} else {primary_rank_factor = ((float) 1);@@ -1126,7 +1127,7 @@ void Collection::score_results(const std::vector<sort_by> & sort_fields, const i// initialize secondary_rank_factorfield sort_field = sort_schema.at(sort_fields[1].name);- if(sort_field.is_integer()) {+ if(sort_field.is_single_integer()) {secondary_rank_factor = ((int64_t) 1);} else {secondary_rank_factor = ((float) 1);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a1655486..c0bc9380 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -9,28 +9,14 @@ CollectionManager::CollectionManager() {}Collection* CollectionManager::init_collection(const nlohmann::json & collection_meta,- const uint32_t collection_next_seq_id) {+ const uint32_t collection_next_seq_id) {std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();- std::vector<field> search_fields;- nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];+ std::vector<field> fields;+ nlohmann::json fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];- for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {- search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});- }-- std::vector<field> facet_fields;- nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];-- for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {- facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});- }-- std::vector<field> collection_sort_fields;- nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];-- for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {- collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});+ for (nlohmann::json::iterator it = fields_map.begin(); it != fields_map.end(); ++it) {+ fields.push_back({it.value()[fields::name], it.value()[fields::type], it.value()[fields::facet]});}std::string token_ranking_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();@@ -39,9 +25,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectioncollection_meta[COLLECTION_ID_KEY].get<uint32_t>(),collection_next_seq_id,store,- search_fields,- facet_fields,- collection_sort_fields,+ fields,token_ranking_field);return collection;@@ -128,49 +112,29 @@ bool CollectionManager::auth_key_matches(std::string auth_key_sent) {return (auth_key == auth_key_sent);}-Option<Collection*> CollectionManager::create_collection(std::string name, const std::vector<field> & search_fields,- const std::vector<field> & facet_fields,- const std::vector<field> & sort_fields,- const std::string & token_ranking_field) {+Option<Collection*> CollectionManager::create_collection(std::string name, const std::vector<field> & fields,+ const std::string & token_ranking_field) {if(store->contains(Collection::get_meta_key(name))) {return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");}nlohmann::json collection_meta;- nlohmann::json search_fields_json = nlohmann::json::array();;- for(const field & search_field: search_fields) {+ nlohmann::json fields_json = nlohmann::json::array();;+ for(const field & field: fields) {nlohmann::json field_val;- field_val[fields::name] = search_field.name;- field_val[fields::type] = search_field.type;- search_fields_json.push_back(field_val);- }-- nlohmann::json facet_fields_json = nlohmann::json::array();;- for(const field & facet_field: facet_fields) {- nlohmann::json field_val;- field_val[fields::name] = facet_field.name;- field_val[fields::type] = facet_field.type;- facet_fields_json.push_back(field_val);- }-- nlohmann::json sort_fields_json = nlohmann::json::array();;- for(const field & sort_field: sort_fields) {- nlohmann::json sort_field_val;- sort_field_val[fields::name] = sort_field.name;- sort_field_val[fields::type] = sort_field.type;- sort_fields_json.push_back(sort_field_val);+ field_val[fields::name] = field.name;+ field_val[fields::type] = field.type;+ field_val[fields::facet] = field.facet;+ fields_json.push_back(field_val);}collection_meta[COLLECTION_NAME_KEY] = name;collection_meta[COLLECTION_ID_KEY] = next_collection_id;- collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = search_fields_json;- collection_meta[COLLECTION_FACET_FIELDS_KEY] = facet_fields_json;- collection_meta[COLLECTION_SORT_FIELDS_KEY] = sort_fields_json;+ collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY] = token_ranking_field;- Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, facet_fields,- sort_fields, token_ranking_field);+ Collection* new_collection = new Collection(name, next_collection_id, 0, store, fields, token_ranking_field);next_collection_id++;store->insert(Collection::get_next_seq_id_key(name), std::to_string(0));diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex 4a3bef22..fdf5e8fc 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -16,15 +16,16 @@ using namespace std;int main(int argc, char* argv[]) {system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");- std::vector<field> fields_to_index = {field("title", field_types::STRING)};- std::vector<field> sort_fields = { field("points", field_types::INT32)};+ std::vector<field> fields_to_index = { field("title", field_types::STRING, false),+ field("points", field_types::INT32, false) };+Store *store = new Store("/tmp/typesense-data");CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(store, "abcd");Collection *collection = collectionManager.get_collection("hnstories_direct");if(collection == nullptr) {- collection = collectionManager.create_collection("hnstories_direct", fields_to_index, {}, sort_fields, "points").get();+ collection = collectionManager.create_collection("hnstories_direct", fields_to_index, "points").get();}std::ifstream infile("/Users/kishore/Downloads/hnstories_tiny.jsonl");diff --git a/src/main/main.cpp b/src/main/main.cppindex 75adce40..0b3a027c 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -24,27 +24,17 @@ int main(int argc, char* argv[]) {collectionManager.init(store, "abcd");std::vector<field> fields_to_index = {- field("lang", field_types::STRING),- field("description", field_types::STRING),- field("topics", field_types::STRING_ARRAY),- field("stars", field_types::INT32),- field("repo_name", field_types::STRING),- field("org", field_types::STRING)- };-- std::vector<field> facet_fields_index = {- field("lang", field_types::STRING),- field("org", field_types::STRING),- field("topics", field_types::STRING_ARRAY)- };-- std::vector<field> sort_fields = {- field("stars", field_types::INT32)+ field("lang", field_types::STRING, true),+ field("description", field_types::STRING, false),+ field("topics", field_types::STRING_ARRAY, true),+ field("stars", field_types::INT32, false),+ field("repo_name", field_types::STRING, false),+ field("org", field_types::STRING, true)};Collection *collection = collectionManager.get_collection("github_top1k");if(collection == nullptr) {- collection = collectionManager.create_collection("github_top1k", fields_to_index, facet_fields_index, sort_fields).get();+ collection = collectionManager.create_collection("github_top1k", fields_to_index).get();}int j = 0;diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex e49cd459..adac570a 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -11,9 +11,6 @@ protected:CollectionManager & collectionManager = CollectionManager::get_instance();Collection *collection1;std::vector<field> search_fields;- std::vector<field> facet_fields;- std::vector<field> sort_fields_index;-std::vector<sort_by> sort_fields;void setupCollection() {@@ -24,13 +21,15 @@ protected:store = new Store(state_dir_path);collectionManager.init(store, "auth_key");- search_fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};- facet_fields = {field("starring", field_types::STRING)};- sort_fields = { sort_by("points", "DESC") };- sort_fields_index = { field("points", field_types::INT32) };+ search_fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("cast", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false)+ };- collection1 = collectionManager.create_collection("collection1", search_fields, facet_fields,- sort_fields_index, "points").get();+ sort_fields = { sort_by("points", "DESC") };+ collection1 = collectionManager.create_collection("collection1", search_fields, "points").get();}virtual void SetUp() {@@ -49,7 +48,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {ASSERT_NE(nullptr, collection1);spp::sparse_hash_map<std::string, field> schema = collection1->get_schema();- std::vector<std::string> facet_fields_expected = {facet_fields[0].name};+ std::vector<std::string> facet_fields_expected = {"cast"};ASSERT_EQ(0, collection1->get_collection_id());ASSERT_EQ(0, collection1->get_next_seq_id());@@ -79,11 +78,11 @@ TEST_F(CollectionManagerTest, CollectionCreation) {ASSERT_EQ(3, num_keys);ASSERT_EQ("1", next_seq_id); // we already call `collection1->get_next_seq_id` above, which is side-effecting- ASSERT_EQ("{\"facet_fields\":[{\"name\":\"starring\",\"type\":\"string\"}],\"id\":0,\"name\":\"collection1\","- "\"search_fields\":[{\"name\":\"title\",\"type\":\"string\"},"- "{\"name\":\"starring\",\"type\":\"string\"}],"- "\"sort_fields\":[{\"name\":\"points\",\"type\":\"int32\"}],"- "\"token_ranking_field\":\"points\"}", collection_meta_json);+ ASSERT_EQ("{\"fields\":[{\"facet\":false,\"name\":\"title\",\"type\":\"string\"},"+ "{\"facet\":false,\"name\":\"starring\",\"type\":\"string\"},"+ "{\"facet\":true,\"name\":\"cast\",\"type\":\"string[]\"},"+ "{\"facet\":false,\"name\":\"points\",\"type\":\"int32\"}"+ "],\"id\":0,\"name\":\"collection1\",\"token_ranking_field\":\"points\"}", collection_meta_json);ASSERT_EQ("1", next_collection_id);}@@ -93,8 +92,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) {ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());// try creating one more collection- collectionManager.create_collection("collection2", search_fields, facet_fields,- sort_fields_index, "points");+ collectionManager.create_collection("collection2", search_fields, "points");collection_vec = collectionManager.get_collections();ASSERT_EQ(2, collection_vec.size());@@ -130,7 +128,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {collection1 = collectionManager2.get_collection("collection1");ASSERT_NE(nullptr, collection1);- std::vector<std::string> facet_fields_expected = {facet_fields[0].name};+ std::vector<std::string> facet_fields_expected = {"cast"};ASSERT_EQ(0, collection1->get_collection_id());ASSERT_EQ(18, collection1->get_next_seq_id());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d0690e1a..45819e99 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -14,8 +14,6 @@ protected:std::vector<std::string> query_fields;Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();- std::vector<field> facet_fields;- std::vector<field> sort_fields_index;std::vector<sort_by> sort_fields;void setupCollection() {@@ -27,17 +25,17 @@ protected:collectionManager.init(store, "auth_key");std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");- std::vector<field> search_fields = {field("title", field_types::STRING)};+ std::vector<field> search_fields = {+ field("title", field_types::STRING, false),+ field("points", field_types::INT32, false)+ };query_fields = {"title"};- facet_fields = { };sort_fields = { sort_by("points", "DESC") };- sort_fields_index = { field("points", field_types::INT32) };collection = collectionManager.get_collection("collection");if(collection == nullptr) {- collection = collectionManager.create_collection("collection", search_fields, facet_fields,- sort_fields_index, "points").get();+ collection = collectionManager.create_collection("collection", search_fields, "points").get();}std::string json_line;@@ -472,12 +470,16 @@ TEST_F(CollectionTest, MultipleFields) {Collection *coll_mul_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");- std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),- field("cast", field_types::STRING_ARRAY)};+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("cast", field_types::STRING_ARRAY, false),+ field("points", field_types::INT32, false)+ };coll_mul_fields = collectionManager.get_collection("coll_mul_fields");if(coll_mul_fields == nullptr) {- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index).get();+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields).get();}std::string json_line;@@ -548,19 +550,69 @@ TEST_F(CollectionTest, MultipleFields) {collectionManager.drop_collection("coll_mul_fields");}+TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {+ Collection *coll_mul_fields;++ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("cast", field_types::STRING_ARRAY, true),+ field("points", field_types::INT32, false)+ };++ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");+ if(coll_mul_fields == nullptr) {+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields).get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ coll_mul_fields->add(json_line);+ }++ infile.close();++ std::vector<std::string> facets;++ // query should be allowed only on non-faceted text fields+ query_fields = {"cast"};+ Option<nlohmann::json> result_op =+ coll_mul_fields->search("anton", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_EQ(false, result_op.ok());+ ASSERT_EQ(400, result_op.code());+ ASSERT_EQ("Field `cast` is a faceted field - it cannot be used as a query field.", result_op.error());++ // only faceted fields should be allowed for filtering+ query_fields = {"title"};+ result_op = coll_mul_fields->search("captain", query_fields, "starring: Samuel", facets, sort_fields, 0, 10, 1,+ FREQUENCY, false);+ ASSERT_EQ(false, result_op.ok());+ ASSERT_EQ(400, result_op.code());+ ASSERT_EQ("Field `starring` is not a faceted field - only a field marked as a facet can be filtered on.",+ result_op.error());++ collectionManager.drop_collection("coll_mul_fields");+}+TEST_F(CollectionTest, FilterOnNumericFields) {Collection *coll_array_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),- field("years", field_types::INT32_ARRAY),- field("timestamps", field_types::INT64_ARRAY)};+ std::vector<field> fields = {+ field("name", field_types::STRING, false),+ field("age", field_types::INT32, false),+ field("years", field_types::INT32_ARRAY, false),+ field("timestamps", field_types::INT64_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, true)+ };+std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };- std::vector<field> sort_fields_index = { field("age", field_types::INT32) };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();}std::string json_line;@@ -688,16 +740,18 @@ TEST_F(CollectionTest, FilterOnFloatFields) {Collection *coll_array_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),- field("top_3", field_types::FLOAT_ARRAY),- field("rating", field_types::FLOAT)};- std::vector<field> sort_fields_index = { field("rating", "FLOAT") };+ std::vector<field> fields = {+ field("name", field_types::STRING, false),+ field("age", field_types::INT32, false),+ field("top_3", field_types::FLOAT_ARRAY, false),+ field("rating", field_types::FLOAT, false)+ };std::vector<sort_by> sort_fields_desc = { sort_by("rating", "DESC") };std::vector<sort_by> sort_fields_asc = { sort_by("rating", "ASC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();}std::string json_line;@@ -825,13 +879,17 @@ TEST_F(CollectionTest, SortOnFloatFields) {Collection *coll_float_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/float_documents.jsonl");- std::vector<field> fields = {field("title", field_types::STRING), field("score", field_types::FLOAT)};- std::vector<field> sort_fields_index = { field("score", "FLOAT"), field("average", "FLOAT") };+ std::vector<field> fields = {+ field("title", field_types::STRING, false),+ field("score", field_types::FLOAT, false),+ field("average", field_types::FLOAT, false)+ };+std::vector<sort_by> sort_fields_desc = { sort_by("score", "DESC"), sort_by("average", "DESC") };coll_float_fields = collectionManager.get_collection("coll_float_fields");if(coll_float_fields == nullptr) {- coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, facet_fields, sort_fields_index).get();+ coll_float_fields = collectionManager.create_collection("coll_float_fields", fields).get();}std::string json_line;@@ -891,16 +949,18 @@ TEST_F(CollectionTest, FilterOnTextFields) {Collection *coll_array_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),- field("years", field_types::INT32_ARRAY),- field("tags", field_types::STRING_ARRAY)};+ std::vector<field> fields = {+ field("name", field_types::STRING, false),+ field("age", field_types::INT32, false),+ field("years", field_types::INT32_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, true)+ };- std::vector<field> sort_fields_index = { field("age", field_types::INT32) };std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();}std::string json_line;@@ -972,17 +1032,16 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {Collection *coll_array_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),- field("years", field_types::INT32_ARRAY),- field("timestamps", field_types::INT64_ARRAY),- field("tags", field_types::STRING_ARRAY)};+ std::vector<field> fields = {field("name", field_types::STRING, false), field("age", field_types::INT32, false),+ field("years", field_types::INT32_ARRAY, false),+ field("timestamps", field_types::INT64_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, false)};- std::vector<field> sort_fields_index = { field("age", field_types::INT32) };std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();}std::string json_line;@@ -1027,24 +1086,27 @@ TEST_F(CollectionTest, FacetCounts) {Collection *coll_array_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),- field("years", field_types::INT32_ARRAY),- field("timestamps", field_types::INT64_ARRAY),- field("tags", field_types::STRING_ARRAY)};- facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("name_facet", field_types::STRING, true),+ field("age", field_types::INT32, false),+ field("years", field_types::INT32_ARRAY, false),+ field("timestamps", field_types::INT64_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, true)};- std::vector<field> sort_fields_index = { field("age", "DESC") };std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();}std::string json_line;while (std::getline(infile, json_line)) {- coll_array_fields->add(json_line);+ nlohmann::json document = nlohmann::json::parse(json_line);+ document["name_facet"] = document["name"];+ const std::string & patched_json_line = document.dump();+ coll_array_fields->add(patched_json_line);}infile.close();@@ -1072,14 +1134,14 @@ TEST_F(CollectionTest, FacetCounts) {// 2 facets, 1 text filter with no filtersfacets.clear();facets.push_back("tags");- facets.push_back("name");+ facets.push_back("name_facet");results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(2, results["facet_counts"].size());ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);- ASSERT_EQ("name", results["facet_counts"][1]["field_name"]);+ ASSERT_EQ("name_facet", results["facet_counts"][1]["field_name"]);// facet value must one that's stored, not indexed (i.e. no tokenization/standardization)ASSERT_EQ("Jeremy Howard", results["facet_counts"][1]["counts"][0]["value"]);@@ -1109,12 +1171,14 @@ TEST_F(CollectionTest, SortingOrder) {Collection *coll_mul_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");- std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),- field("cast", field_types::STRING_ARRAY)};+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("points", field_types::INT32, false),+ field("cast", field_types::STRING_ARRAY, false)};coll_mul_fields = collectionManager.get_collection("coll_mul_fields");if(coll_mul_fields == nullptr) {- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index).get();+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields).get();}std::string json_line;@@ -1192,17 +1256,17 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {Collection *coll_array_fields;std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),- field("years", field_types::INT32_ARRAY),- field("timestamps", field_types::INT64_ARRAY),- field("tags", field_types::STRING_ARRAY)};- facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};- std::vector<field> sort_fields_index = { field("age", "DESC") };+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("age", field_types::INT32, false),+ field("years", field_types::INT32_ARRAY, false),+ field("timestamps", field_types::INT64_ARRAY, false),+ field("tags", field_types::STRING_ARRAY, true)};+std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };coll_array_fields = collectionManager.get_collection("coll_array_fields");if(coll_array_fields == nullptr) {- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();}std::string json_line;@@ -1220,12 +1284,12 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {Option<nlohmann::json> res_op = coll_array_fields->search("the", query_fields_not_found, "", facets, sort_fields, 0, 10);ASSERT_FALSE(res_op.ok());ASSERT_EQ(400, res_op.code());- ASSERT_STREQ("Could not find a search field named `titlez` in the schema.", res_op.error().c_str());+ ASSERT_STREQ("Could not find a field named `titlez` in the schema.", res_op.error().c_str());// when a query field is an integer fieldres_op = coll_array_fields->search("the", {"age"}, "", facets, sort_fields, 0, 10);ASSERT_EQ(400, res_op.code());- ASSERT_STREQ("Search field `age` should be a string or a string array.", res_op.error().c_str());+ ASSERT_STREQ("Field `age` should be a string or a string array.", res_op.error().c_str());// when a facet field is not defined in the schemares_op = coll_array_fields->search("the", {"name"}, "", {"timestamps"}, sort_fields, 0, 10);@@ -1235,11 +1299,11 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {// when a rank field is not defined in the schemares_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("timestamps", "ASC") }, 0, 10);ASSERT_EQ(400, res_op.code());- ASSERT_STREQ("Could not find a sort field named `timestamps` in the schema.", res_op.error().c_str());+ ASSERT_STREQ("Could not find a field named `timestamps` in the schema.", res_op.error().c_str());res_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("_rank", "ASC") }, 0, 10);ASSERT_EQ(400, res_op.code());- ASSERT_STREQ("Could not find a sort field named `_rank` in the schema.", res_op.error().c_str());+ ASSERT_STREQ("Could not find a field named `_rank` in the schema.", res_op.error().c_str());collectionManager.drop_collection("coll_array_fields");}@@ -1248,45 +1312,45 @@ TEST_F(CollectionTest, IndexingWithBadData) {// should not crash when document to-be-indexed doesn't match schemaCollection *sample_collection;- std::vector<field> fields = {field("name", field_types::STRING)};- facet_fields = {field("tags", field_types::STRING_ARRAY)};+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("tags", field_types::STRING_ARRAY, true),+ field("age", field_types::INT32, false),+ field("average", field_types::INT32, false) };- std::vector<field> sort_fields_index = { field("age", field_types::INT32), field("average", field_types::INT32) };std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };sample_collection = collectionManager.get_collection("sample_collection");if(sample_collection == nullptr) {- sample_collection = collectionManager.create_collection("sample_collection", fields, facet_fields,- sort_fields_index, "age").get();+ sample_collection = collectionManager.create_collection("sample_collection", fields, "age").get();}- const Option<std::string> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29}");+ const Option<std::string> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29, \"average\": 78}");ASSERT_FALSE(search_fields_missing_op1.ok());- ASSERT_STREQ("Field `name` has been declared as a search field in the schema, but is not found in the document.",+ ASSERT_STREQ("Field `name` has been declared in the schema, but is not found in the document.",search_fields_missing_op1.error().c_str());- const Option<std::string> & search_fields_missing_op2 = sample_collection->add("{\"namez\": \"foo\", \"age\": 34}");+ const Option<std::string> & search_fields_missing_op2 = sample_collection->add("{\"namez\": \"foo\", \"age\": 34, \"average\": 78}");ASSERT_FALSE(search_fields_missing_op2.ok());- ASSERT_STREQ("Field `name` has been declared as a search field in the schema, but is not found in the document.",+ ASSERT_STREQ("Field `name` has been declared in the schema, but is not found in the document.",search_fields_missing_op2.error().c_str());- const Option<std::string> & facet_fields_missing_op1 = sample_collection->add("{\"name\": \"foo\", \"age\": 34}");+ const Option<std::string> & facet_fields_missing_op1 = sample_collection->add("{\"name\": \"foo\", \"age\": 34, \"average\": 78}");ASSERT_FALSE(facet_fields_missing_op1.ok());- ASSERT_STREQ("Field `tags` has been declared as a facet field in the schema, but is not found in the document.",+ ASSERT_STREQ("Field `tags` has been declared in the schema, but is not found in the document.",facet_fields_missing_op1.error().c_str());- const char *doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", \"blue\"]}";+ const char *doc_str = "{\"name\": \"foo\", \"age\": 34, \"avg\": 78, \"tags\": [\"red\", \"blue\"]}";const Option<std::string> & sort_fields_missing_op1 = sample_collection->add(doc_str);ASSERT_FALSE(sort_fields_missing_op1.ok());- ASSERT_STREQ("Field `average` has been declared as a sort field in the schema, but is not found in the document.",+ ASSERT_STREQ("Field `average` has been declared in the schema, but is not found in the document.",sort_fields_missing_op1.error().c_str());// Handle type errors- doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22}";+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22, \"average\": 78}";const Option<std::string> & bad_facet_field_op = sample_collection->add(doc_str);ASSERT_FALSE(bad_facet_field_op.ok());- ASSERT_STREQ("Facet field `tags` must be a string array.", bad_facet_field_op.error().c_str());+ ASSERT_STREQ("Field `tags` must be a string array.", bad_facet_field_op.error().c_str());doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";const Option<std::string> & empty_facet_field_op = sample_collection->add(doc_str);@@ -1311,7 +1375,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";const Option<std::string> & bad_rank_field_op = sample_collection->add(doc_str);ASSERT_FALSE(bad_rank_field_op.ok());- ASSERT_STREQ("Sort field `average` must be a number.", bad_rank_field_op.error().c_str());+ ASSERT_STREQ("Field `average` must be an int32.", bad_rank_field_op.error().c_str());doc_str = "{\"name\": \"foo\", \"age\": asdadasd, \"tags\": [], \"average\": 34 }";const Option<std::string> & bad_token_ranking_field_op4 = sample_collection->add(doc_str);@@ -1324,15 +1388,16 @@ TEST_F(CollectionTest, IndexingWithBadData) {TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {Collection *empty_coll;- std::vector<field> fields = {field("name", field_types::STRING)};- facet_fields = {field("tags", field_types::STRING_ARRAY)};+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("tags", field_types::STRING_ARRAY, false),+ field("age", field_types::INT32, false),+ field("average", field_types::INT32, false)};- std::vector<field> sort_fields_index = { field("age", field_types::INT32), field("average", field_types::INT32) };std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };empty_coll = collectionManager.get_collection("empty_coll");if(empty_coll == nullptr) {- empty_coll = collectionManager.create_collection("empty_coll", fields, facet_fields, sort_fields_index, "age").get();+ empty_coll = collectionManager.create_collection("empty_coll", fields, "age").get();}nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();@@ -1343,15 +1408,16 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {TEST_F(CollectionTest, IdFieldShouldBeAString) {Collection *coll1;- std::vector<field> fields = {field("name", field_types::STRING)};- facet_fields = {field("tags", field_types::STRING_ARRAY)};+ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("tags", field_types::STRING_ARRAY, false),+ field("age", field_types::INT32, false),+ field("average", field_types::INT32, false)};- std::vector<field> sort_fields_index = { field("age", field_types::INT32), field("average", field_types::INT32) };std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };coll1 = collectionManager.get_collection("coll1");if(coll1 == nullptr) {- coll1 = collectionManager.create_collection("coll1", fields, facet_fields, sort_fields_index, "age").get();+ coll1 = collectionManager.create_collection("coll1", fields, "age").get();}nlohmann::json doc;@@ -1373,17 +1439,18 @@ TEST_F(CollectionTest, DeletionOfADocument) {collectionManager.drop_collection("collection");std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");- std::vector<field> search_fields = {field("title", field_types::STRING)};++ std::vector<field> search_fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false)};++std::vector<std::string> query_fields = {"title"};- std::vector<field> facet_fields = { };std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };- std::vector<field> sort_fields_index = { field("points", field_types::INT32) };Collection *collection_for_del;collection_for_del = collectionManager.get_collection("collection_for_del");if(collection_for_del == nullptr) {- collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, facet_fields,- sort_fields_index, "points").get();+ collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, "points").get();}std::string json_line;