Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Combine search, facet and sort fields into a single "fields".

Kishore Nallan0e4517d9019c
diff --git a/TODO.md b/TODO.md
index 1a483ad9..01240c98 100644
--- a/TODO.md
+++ b/TODO.md
@@ -67,6 +67,11 @@
- ~~test for string filter comparison: title < "foo"~~
- ~~Test for sorted_array::indexOf when length is 0~~
- ~~Test for pagination~~
+- ~~search_fields, sort_fields and facet fields should be combined~~
+- ~~facet fields should be indexed verbatim~~
+- change "search_by" to "query_by"
+- during index_in_memory() validations should be front loaded
+- NOT operator support
- test for token ranking on float field
- test for float int field deletion during doc deletion
- Test for snippets
diff --git a/include/collection.h b/include/collection.h
index b6bf155e..e499ef5c 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -123,8 +123,7 @@ public:
Collection() = delete;
Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> & search_fields, const std::vector<field> & facet_fields,
- const std::vector<field> & sort_fields, const std::string token_ranking_field);
+ const std::vector<field> & fields, const std::string & token_ranking_field);
~Collection();
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 479c1a0f..dada354f 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -22,9 +22,7 @@ private:
static constexpr const char* COLLECTION_NAME_KEY = "name";
static constexpr const char* COLLECTION_ID_KEY = "id";
- static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "search_fields";
- static constexpr const char* COLLECTION_FACET_FIELDS_KEY = "facet_fields";
- static constexpr const char* COLLECTION_SORT_FIELDS_KEY = "sort_fields";
+ static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields";
static constexpr const char* COLLECTION_TOKEN_ORDERING_FIELD_KEY = "token_ranking_field";
std::string auth_key;
@@ -53,10 +51,8 @@ public:
bool auth_key_matches(std::string auth_key_sent);
- Option<Collection*> create_collection(std::string name, const std::vector<field> & search_fields,
- const std::vector<field> & facet_fields,
- const std::vector<field> & sort_fields,
- const std::string & token_ranking_field = "");
+ Option<Collection*> create_collection(std::string name, const std::vector<field> & fields,
+ const std::string & token_ranking_field = "");
Collection* get_collection(const std::string & collection_name);
diff --git a/include/field.h b/include/field.h
index 4408c400..f983e0df 100644
--- a/include/field.h
+++ b/include/field.h
@@ -19,28 +19,42 @@ namespace field_types {
namespace fields {
static const std::string name = "name";
static const std::string type = "type";
+ static const std::string facet = "facet";
}
struct field {
std::string name;
std::string type;
+ bool facet;
- field(const std::string & name, const std::string & type): name(name), type(type) {
+ field(const std::string & name, const std::string & type, const bool & facet): name(name), type(type), facet(facet) {
}
- bool is_integer() {
+ bool is_single_integer() const {
+ return (type == field_types::INT32 || type == field_types::INT64);
+ }
+
+ bool is_single_float() const {
+ return (type == field_types::FLOAT);
+ }
+
+ bool is_integer() const {
return (type == field_types::INT32 || type == field_types::INT32_ARRAY ||
type == field_types::INT64 || type == field_types::INT64_ARRAY);
}
- bool is_float() {
+ bool is_float() const {
return (type == field_types::FLOAT || type == field_types::FLOAT_ARRAY);
}
- bool is_string() {
+ bool is_string() const {
return (type == field_types::STRING || type == field_types::STRING_ARRAY);
}
+
+ bool is_facet() const {
+ return facet;
+ }
};
struct filter {
diff --git a/src/api.cpp b/src/api.cpp
index 46b6ef54..93619a40 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -40,12 +40,8 @@ void post_create_collection(http_req & req, http_res & res) {
return res.send_400("Parameter `name` is required.");
}
- if(req_json.count("search_fields") == 0) {
- return res.send_400("Parameter `search_fields` is required.");
- }
-
- if(req_json.count("sort_fields") == 0) {
- return res.send_400("Parameter `sort_fields` is required.");
+ if(req_json.count("fields") == 0) {
+ return res.send_400("Parameter `fields` is required.");
}
if(collectionManager.get_collection(req_json["name"]) != nullptr) {
@@ -54,69 +50,29 @@ void post_create_collection(http_req & req, http_res & res) {
// field specific validation
- std::vector<field> search_fields;
+ std::vector<field> fields;
- if(!req_json["search_fields"].is_array() || req_json["search_fields"].size() == 0) {
- return res.send_400("Wrong format for `search_fields`. It should be an array like: "
+ if(!req_json["fields"].is_array() || req_json["fields"].size() == 0) {
+ return res.send_400("Wrong format for `fields`. It should be an array like: "
"[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
}
- for(const nlohmann::json & search_field_json: req_json["search_fields"]) {
- if(!search_field_json.is_object() ||
- search_field_json.count(fields::name) == 0 || search_field_json.count(fields::type) == 0 ||
- !search_field_json.at(fields::name).is_string() || !search_field_json.at(fields::type).is_string()) {
+ for(nlohmann::json & field_json: req_json["fields"]) {
+ if(!field_json.is_object() ||
+ field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 ||
+ !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) {
- return res.send_400("Wrong format for `search_fields`. It should be an array like: "
+ return res.send_400("Wrong format for `fields`. It should be an array like: "
"[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
}
- search_fields.push_back(field(search_field_json["name"], search_field_json["type"]));
- }
-
- std::vector<field> facet_fields;
-
- if(req_json.count("facet_fields") != 0) {
- if(!req_json["facet_fields"].is_array()) {
- return res.send_400("Wrong format for `facet_fields`. It should be an array like: "
- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
- }
-
- for(const nlohmann::json & facet_field_json: req_json["facet_fields"]) {
- if(!facet_field_json.is_object() ||
- facet_field_json.count(fields::name) == 0 || facet_field_json.count(fields::type) == 0 ||
- !facet_field_json.at(fields::name).is_string() || !facet_field_json.at(fields::type).is_string()) {
-
- return res.send_400("Wrong format for `facet_fields`. It should be an array like: "
- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
- }
-
- facet_fields.push_back(field(facet_field_json["name"], facet_field_json["type"]));
- }
- }
-
- std::vector<field> sort_fields;
-
- if(!req_json["sort_fields"].is_array() || req_json["sort_fields"].size() == 0) {
- return res.send_400("Wrong format for `sort_fields`. It should be an array like: "
- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
- }
-
- for(const nlohmann::json & sort_field_json: req_json["sort_fields"]) {
- if(!sort_field_json.is_object() ||
- sort_field_json.count(fields::name) == 0 || sort_field_json.count(fields::type) == 0 ||
- !sort_field_json.at(fields::name).is_string() ||
- !sort_field_json.at(fields::type).is_string()) {
-
- return res.send_400("Wrong format for `sort_fields`. It should be an array like: "
- "[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");
- }
-
- const std::string & sort_field_type = sort_field_json["type"].get<std::string>();
- if(sort_field_type != field_types::INT32 && sort_field_type != field_types::INT64) {
- return res.send_400("Sort field `" + sort_field_json["name"].get<std::string>() + "` must be an integer.");
+ if(field_json.count("facet") == 0) {
+ field_json["facet"] = false;
}
- sort_fields.push_back(field(sort_field_json["name"], sort_field_json["type"]));
+ fields.push_back(
+ field(field_json["name"], field_json["type"], field_json["facet"])
+ );
}
const char* PREFIX_RANKING_FIELD = "prefix_ranking_field";
@@ -124,13 +80,14 @@ void post_create_collection(http_req & req, http_res & res) {
if(req_json.count(PREFIX_RANKING_FIELD) != 0) {
if(!req_json[PREFIX_RANKING_FIELD].is_string()) {
- return res.send_400(std::string("`") + PREFIX_RANKING_FIELD + "` should be the name of an unsigned integer field.");
+ return res.send_400(std::string("`") + PREFIX_RANKING_FIELD +
+ "` should be a string. It should be the name of an unsigned integer field.");
}
token_ranking_field = req_json[PREFIX_RANKING_FIELD].get<std::string>();
}
- collectionManager.create_collection(req_json["name"], search_fields, facet_fields, sort_fields, token_ranking_field);
+ collectionManager.create_collection(req_json["name"], fields, token_ranking_field);
res.send_201(req.body);
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 318b12c9..0c1789b0 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -8,28 +8,27 @@
#include <art.h>
Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
- const std::vector<field> &search_fields, const std::vector<field> & facet_fields,
- const std::vector<field> & sort_fields, const std::string token_ranking_field):
+ const std::vector<field> &fields, const std::string & token_ranking_field):
name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
token_ranking_field(token_ranking_field) {
- for(const field& field: search_fields) {
+ for(const field& field: fields) {
art_tree *t = new art_tree;
art_tree_init(t);
search_index.emplace(field.name, t);
search_schema.emplace(field.name, field);
- }
- for(const field& field: facet_fields) {
- facet_value fvalue;
- facet_index.emplace(field.name, fvalue);
- facet_schema.emplace(field.name, field);
- }
+ if(field.is_facet()) {
+ facet_value fvalue;
+ facet_index.emplace(field.name, fvalue);
+ facet_schema.emplace(field.name, field);
+ }
- for(const field & sort_field: sort_fields) {
- spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
- sort_index.emplace(sort_field.name, doc_to_score);
- sort_schema.emplace(sort_field.name, sort_field);
+ if(field.is_single_integer() || field.is_single_float()) {
+ spp::sparse_hash_map<uint32_t, number_t> * doc_to_score = new spp::sparse_hash_map<uint32_t, number_t>();
+ sort_index.emplace(field.name, doc_to_score);
+ sort_schema.emplace(field.name, field);
+ }
}
num_documents = 0;
@@ -123,7 +122,7 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
const std::string & field_name = field_pair.first;
if(document.count(field_name) == 0) {
- return Option<>(400, "Field `" + field_name + "` has been declared as a search field in the schema, "
+ return Option<>(400, "Field `" + field_name + "` has been declared in the schema, "
"but is not found in the document.");
}
@@ -131,80 +130,92 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
if(field_pair.second.type == field_types::STRING) {
if(!document[field_name].is_string()) {
- return Option<>(400, "Search field `" + field_name + "` must be a string.");
+ return Option<>(400, "Field `" + field_name + "` must be a string.");
}
const std::string & text = document[field_name];
- index_string_field(text, points, t, seq_id, false);
+ index_string_field(text, points, t, seq_id, field_pair.second.is_facet());
} else if(field_pair.second.type == field_types::INT32) {
if(!document[field_name].is_number_integer()) {
- return Option<>(400, "Search field `" + field_name + "` must be an int32.");
+ return Option<>(400, "Field `" + field_name + "` must be an int32.");
}
if(document[field_name].get<int64_t>() > INT32_MAX) {
- return Option<>(400, "Search field `" + field_name + "` exceeds maximum value of int32.");
+ return Option<>(400, "Field `" + field_name + "` exceeds maximum value of int32.");
}
uint32_t value = document[field_name];
index_int32_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::INT64) {
if(!document[field_name].is_number_integer()) {
- return Option<>(400, "Search field `" + field_name + "` must be an int64.");
+ return Option<>(400, "Field `" + field_name + "` must be an int64.");
}
uint64_t value = document[field_name];
index_int64_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::FLOAT) {
if(!document[field_name].is_number_float()) {
- return Option<>(400, "Search field `" + field_name + "` must be a float.");
+ return Option<>(400, "Field `" + field_name + "` must be a float.");
}
float value = document[field_name];
index_float_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
if(!document[field_name].is_array()) {
- return Option<>(400, "Search field `" + field_name + "` must be a string array.");
+ return Option<>(400, "Field `" + field_name + "` must be a string array.");
}
if(document[field_name].size() > 0 && !document[field_name][0].is_string()) {
- return Option<>(400, "Search field `" + field_name + "` must be a string array.");
+ return Option<>(400, "Field `" + field_name + "` must be a string array.");
}
std::vector<std::string> strings = document[field_name];
- index_string_array_field(strings, points, t, seq_id, false);
+ index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());
} else if(field_pair.second.type == field_types::INT32_ARRAY) {
if(!document[field_name].is_array()) {
- return Option<>(400, "Search field `" + field_name + "` must be an int32 array.");
+ return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
}
if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
- return Option<>(400, "Search field `" + field_name + "` must be an int32 array.");
+ return Option<>(400, "Field `" + field_name + "` must be an int32 array.");
}
std::vector<int32_t> values = document[field_name];
index_int32_array_field(values, points, t, seq_id);
} else if(field_pair.second.type == field_types::INT64_ARRAY) {
if(!document[field_name].is_array()) {
- return Option<>(400, "Search field `" + field_name + "` must be an int64 array.");
+ return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
}
if(document[field_name].size() > 0 && !document[field_name][0].is_number_integer()) {
- return Option<>(400, "Search field `" + field_name + "` must be an int64 array.");
+ return Option<>(400, "Field `" + field_name + "` must be an int64 array.");
}
std::vector<int64_t> values = document[field_name];
index_int64_array_field(values, points, t, seq_id);
} else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
if(!document[field_name].is_array()) {
- return Option<>(400, "Search field `" + field_name + "` must be a float array.");
+ return Option<>(400, "Field `" + field_name + "` must be a float array.");
}
if(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) {
- return Option<>(400, "Search field `" + field_name + "` must be a float array.");
+ return Option<>(400, "Field `" + field_name + "` must be a float array.");
}
std::vector<float> values = document[field_name];
index_float_array_field(values, points, t, seq_id);
}
+
+ // add numerical values automatically into sort index
+ if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
+ field_pair.second.type == field_types::FLOAT) {
+ spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);
+
+ if(document[field_pair.first].is_number_integer()) {
+ doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());
+ } else if(document[field_pair.first].is_number_float()) {
+ doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());
+ }
+ }
}
for(const std::pair<std::string, field> & field_pair: facet_schema) {
@@ -233,27 +244,8 @@ Option<uint32_t> Collection::index_in_memory(const nlohmann::json &document, uin
const std::vector<std::string> & values = document[field_name];
fvalue.index_values(seq_id, values);
- }
- }
-
- for(const std::pair<std::string, field> & field_pair: sort_schema) {
- const field & sort_field = field_pair.second;
-
- if(document.count(sort_field.name) == 0) {
- return Option<>(400, "Field `" + sort_field.name + "` has been declared as a sort field in the schema, "
- "but is not found in the document.");
- }
-
- if(!document[sort_field.name].is_number()) {
- return Option<>(400, "Sort field `" + sort_field.name + "` must be a number.");
- }
-
- spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(sort_field.name);
-
- if(document[sort_field.name].is_number_integer()) {
- doc_to_score->emplace(seq_id, document[sort_field.name].get<int64_t>());
} else {
- doc_to_score->emplace(seq_id, document[sort_field.name].get<float>());
+ return Option<>(400, "Facet field `" + field_name + "` must be a string or a string[].");
}
}
@@ -598,6 +590,10 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
f = {field_name, {filter_value}, op_comparator.get()};
}
} else if(_field.is_string()) {
+ if(!_field.facet) {
+ return Option<>(400, "Field `" + _field.name + "` is not a faceted field - only a field marked as a "
+ "facet can be filtered on.");
+ }
if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
std::vector<std::string> filter_values;
StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
@@ -678,13 +674,18 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// validate search fields
for(const std::string & field_name: search_fields) {
if(search_schema.count(field_name) == 0) {
- std::string error = "Could not find a search field named `" + field_name + "` in the schema.";
+ std::string error = "Could not find a field named `" + field_name + "` in the schema.";
return Option<nlohmann::json>(400, error);
}
field search_field = search_schema.at(field_name);
if(search_field.type != field_types::STRING && search_field.type != field_types::STRING_ARRAY) {
- std::string error = "Search field `" + field_name + "` should be a string or a string array.";
+ std::string error = "Field `" + field_name + "` should be a string or a string array.";
+ return Option<nlohmann::json>(400, error);
+ }
+
+ if(search_field.facet) {
+ std::string error = "Field `" + field_name + "` is a faceted field - it cannot be used as a query field.";
return Option<nlohmann::json>(400, error);
}
}
@@ -704,7 +705,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
for(const sort_by & _sort_field: sort_fields) {
if(sort_index.count(_sort_field.name) == 0) {
- std::string error = "Could not find a sort field named `" + _sort_field.name + "` in the schema.";
+ std::string error = "Could not find a field named `" + _sort_field.name + "` in the schema.";
return Option<nlohmann::json>(400, error);
}
@@ -712,7 +713,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
StringUtils::toupper(sort_order);
if(sort_order != sort_field_const::asc && sort_order != sort_field_const::desc) {
- std::string error = "Order for sort field` " + _sort_field.name + "` should be either ASC or DESC.";
+ std::string error = "Order for field` " + _sort_field.name + "` should be either ASC or DESC.";
return Option<nlohmann::json>(400, error);
}
@@ -1110,7 +1111,7 @@ void Collection::score_results(const std::vector<sort_by> & sort_fields, const i
// initialize primary_rank_factor
field sort_field = sort_schema.at(sort_fields[0].name);
- if(sort_field.is_integer()) {
+ if(sort_field.is_single_integer()) {
primary_rank_factor = ((int64_t) 1);
} else {
primary_rank_factor = ((float) 1);
@@ -1126,7 +1127,7 @@ void Collection::score_results(const std::vector<sort_by> & sort_fields, const i
// initialize secondary_rank_factor
field sort_field = sort_schema.at(sort_fields[1].name);
- if(sort_field.is_integer()) {
+ if(sort_field.is_single_integer()) {
secondary_rank_factor = ((int64_t) 1);
} else {
secondary_rank_factor = ((float) 1);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index a1655486..c0bc9380 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -9,28 +9,14 @@ CollectionManager::CollectionManager() {
}
Collection* CollectionManager::init_collection(const nlohmann::json & collection_meta,
- const uint32_t collection_next_seq_id) {
+ const uint32_t collection_next_seq_id) {
std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();
- std::vector<field> search_fields;
- nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
+ std::vector<field> fields;
+ nlohmann::json fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
- for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {
- search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
- }
-
- std::vector<field> facet_fields;
- nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];
-
- for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {
- facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
- }
-
- std::vector<field> collection_sort_fields;
- nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];
-
- for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {
- collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
+ for (nlohmann::json::iterator it = fields_map.begin(); it != fields_map.end(); ++it) {
+ fields.push_back({it.value()[fields::name], it.value()[fields::type], it.value()[fields::facet]});
}
std::string token_ranking_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();
@@ -39,9 +25,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
collection_next_seq_id,
store,
- search_fields,
- facet_fields,
- collection_sort_fields,
+ fields,
token_ranking_field);
return collection;
@@ -128,49 +112,29 @@ bool CollectionManager::auth_key_matches(std::string auth_key_sent) {
return (auth_key == auth_key_sent);
}
-Option<Collection*> CollectionManager::create_collection(std::string name, const std::vector<field> & search_fields,
- const std::vector<field> & facet_fields,
- const std::vector<field> & sort_fields,
- const std::string & token_ranking_field) {
+Option<Collection*> CollectionManager::create_collection(std::string name, const std::vector<field> & fields,
+ const std::string & token_ranking_field) {
if(store->contains(Collection::get_meta_key(name))) {
return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");
}
nlohmann::json collection_meta;
- nlohmann::json search_fields_json = nlohmann::json::array();;
- for(const field & search_field: search_fields) {
+ nlohmann::json fields_json = nlohmann::json::array();;
+ for(const field & field: fields) {
nlohmann::json field_val;
- field_val[fields::name] = search_field.name;
- field_val[fields::type] = search_field.type;
- search_fields_json.push_back(field_val);
- }
-
- nlohmann::json facet_fields_json = nlohmann::json::array();;
- for(const field & facet_field: facet_fields) {
- nlohmann::json field_val;
- field_val[fields::name] = facet_field.name;
- field_val[fields::type] = facet_field.type;
- facet_fields_json.push_back(field_val);
- }
-
- nlohmann::json sort_fields_json = nlohmann::json::array();;
- for(const field & sort_field: sort_fields) {
- nlohmann::json sort_field_val;
- sort_field_val[fields::name] = sort_field.name;
- sort_field_val[fields::type] = sort_field.type;
- sort_fields_json.push_back(sort_field_val);
+ field_val[fields::name] = field.name;
+ field_val[fields::type] = field.type;
+ field_val[fields::facet] = field.facet;
+ fields_json.push_back(field_val);
}
collection_meta[COLLECTION_NAME_KEY] = name;
collection_meta[COLLECTION_ID_KEY] = next_collection_id;
- collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = search_fields_json;
- collection_meta[COLLECTION_FACET_FIELDS_KEY] = facet_fields_json;
- collection_meta[COLLECTION_SORT_FIELDS_KEY] = sort_fields_json;
+ collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY] = token_ranking_field;
- Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, facet_fields,
- sort_fields, token_ranking_field);
+ Collection* new_collection = new Collection(name, next_collection_id, 0, store, fields, token_ranking_field);
next_collection_id++;
store->insert(Collection::get_next_seq_id_key(name), std::to_string(0));
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index 4a3bef22..fdf5e8fc 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -16,15 +16,16 @@ using namespace std;
int main(int argc, char* argv[]) {
system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");
- std::vector<field> fields_to_index = {field("title", field_types::STRING)};
- std::vector<field> sort_fields = { field("points", field_types::INT32)};
+ std::vector<field> fields_to_index = { field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false) };
+
Store *store = new Store("/tmp/typesense-data");
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(store, "abcd");
Collection *collection = collectionManager.get_collection("hnstories_direct");
if(collection == nullptr) {
- collection = collectionManager.create_collection("hnstories_direct", fields_to_index, {}, sort_fields, "points").get();
+ collection = collectionManager.create_collection("hnstories_direct", fields_to_index, "points").get();
}
std::ifstream infile("/Users/kishore/Downloads/hnstories_tiny.jsonl");
diff --git a/src/main/main.cpp b/src/main/main.cpp
index 75adce40..0b3a027c 100644
--- a/src/main/main.cpp
+++ b/src/main/main.cpp
@@ -24,27 +24,17 @@ int main(int argc, char* argv[]) {
collectionManager.init(store, "abcd");
std::vector<field> fields_to_index = {
- field("lang", field_types::STRING),
- field("description", field_types::STRING),
- field("topics", field_types::STRING_ARRAY),
- field("stars", field_types::INT32),
- field("repo_name", field_types::STRING),
- field("org", field_types::STRING)
- };
-
- std::vector<field> facet_fields_index = {
- field("lang", field_types::STRING),
- field("org", field_types::STRING),
- field("topics", field_types::STRING_ARRAY)
- };
-
- std::vector<field> sort_fields = {
- field("stars", field_types::INT32)
+ field("lang", field_types::STRING, true),
+ field("description", field_types::STRING, false),
+ field("topics", field_types::STRING_ARRAY, true),
+ field("stars", field_types::INT32, false),
+ field("repo_name", field_types::STRING, false),
+ field("org", field_types::STRING, true)
};
Collection *collection = collectionManager.get_collection("github_top1k");
if(collection == nullptr) {
- collection = collectionManager.create_collection("github_top1k", fields_to_index, facet_fields_index, sort_fields).get();
+ collection = collectionManager.create_collection("github_top1k", fields_to_index).get();
}
int j = 0;
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index e49cd459..adac570a 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -11,9 +11,6 @@ protected:
CollectionManager & collectionManager = CollectionManager::get_instance();
Collection *collection1;
std::vector<field> search_fields;
- std::vector<field> facet_fields;
- std::vector<field> sort_fields_index;
-
std::vector<sort_by> sort_fields;
void setupCollection() {
@@ -24,13 +21,15 @@ protected:
store = new Store(state_dir_path);
collectionManager.init(store, "auth_key");
- search_fields = {field("title", field_types::STRING), field("starring", field_types::STRING)};
- facet_fields = {field("starring", field_types::STRING)};
- sort_fields = { sort_by("points", "DESC") };
- sort_fields_index = { field("points", field_types::INT32) };
+ search_fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("cast", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)
+ };
- collection1 = collectionManager.create_collection("collection1", search_fields, facet_fields,
- sort_fields_index, "points").get();
+ sort_fields = { sort_by("points", "DESC") };
+ collection1 = collectionManager.create_collection("collection1", search_fields, "points").get();
}
virtual void SetUp() {
@@ -49,7 +48,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
ASSERT_NE(nullptr, collection1);
spp::sparse_hash_map<std::string, field> schema = collection1->get_schema();
- std::vector<std::string> facet_fields_expected = {facet_fields[0].name};
+ std::vector<std::string> facet_fields_expected = {"cast"};
ASSERT_EQ(0, collection1->get_collection_id());
ASSERT_EQ(0, collection1->get_next_seq_id());
@@ -79,11 +78,11 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
ASSERT_EQ(3, num_keys);
ASSERT_EQ("1", next_seq_id); // we already call `collection1->get_next_seq_id` above, which is side-effecting
- ASSERT_EQ("{\"facet_fields\":[{\"name\":\"starring\",\"type\":\"string\"}],\"id\":0,\"name\":\"collection1\","
- "\"search_fields\":[{\"name\":\"title\",\"type\":\"string\"},"
- "{\"name\":\"starring\",\"type\":\"string\"}],"
- "\"sort_fields\":[{\"name\":\"points\",\"type\":\"int32\"}],"
- "\"token_ranking_field\":\"points\"}", collection_meta_json);
+ ASSERT_EQ("{\"fields\":[{\"facet\":false,\"name\":\"title\",\"type\":\"string\"},"
+ "{\"facet\":false,\"name\":\"starring\",\"type\":\"string\"},"
+ "{\"facet\":true,\"name\":\"cast\",\"type\":\"string[]\"},"
+ "{\"facet\":false,\"name\":\"points\",\"type\":\"int32\"}"
+ "],\"id\":0,\"name\":\"collection1\",\"token_ranking_field\":\"points\"}", collection_meta_json);
ASSERT_EQ("1", next_collection_id);
}
@@ -93,8 +92,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) {
ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
// try creating one more collection
- collectionManager.create_collection("collection2", search_fields, facet_fields,
- sort_fields_index, "points");
+ collectionManager.create_collection("collection2", search_fields, "points");
collection_vec = collectionManager.get_collections();
ASSERT_EQ(2, collection_vec.size());
@@ -130,7 +128,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
collection1 = collectionManager2.get_collection("collection1");
ASSERT_NE(nullptr, collection1);
- std::vector<std::string> facet_fields_expected = {facet_fields[0].name};
+ std::vector<std::string> facet_fields_expected = {"cast"};
ASSERT_EQ(0, collection1->get_collection_id());
ASSERT_EQ(18, collection1->get_next_seq_id());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d0690e1a..45819e99 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -14,8 +14,6 @@ protected:
std::vector<std::string> query_fields;
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
- std::vector<field> facet_fields;
- std::vector<field> sort_fields_index;
std::vector<sort_by> sort_fields;
void setupCollection() {
@@ -27,17 +25,17 @@ protected:
collectionManager.init(store, "auth_key");
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
- std::vector<field> search_fields = {field("title", field_types::STRING)};
+ std::vector<field> search_fields = {
+ field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)
+ };
query_fields = {"title"};
- facet_fields = { };
sort_fields = { sort_by("points", "DESC") };
- sort_fields_index = { field("points", field_types::INT32) };
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
- collection = collectionManager.create_collection("collection", search_fields, facet_fields,
- sort_fields_index, "points").get();
+ collection = collectionManager.create_collection("collection", search_fields, "points").get();
}
std::string json_line;
@@ -472,12 +470,16 @@ TEST_F(CollectionTest, MultipleFields) {
Collection *coll_mul_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
- std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),
- field("cast", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("cast", field_types::STRING_ARRAY, false),
+ field("points", field_types::INT32, false)
+ };
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index).get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields).get();
}
std::string json_line;
@@ -548,19 +550,69 @@ TEST_F(CollectionTest, MultipleFields) {
collectionManager.drop_collection("coll_mul_fields");
}
+TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
+ Collection *coll_mul_fields;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("cast", field_types::STRING_ARRAY, true),
+ field("points", field_types::INT32, false)
+ };
+
+ coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
+ if(coll_mul_fields == nullptr) {
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields).get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ coll_mul_fields->add(json_line);
+ }
+
+ infile.close();
+
+ std::vector<std::string> facets;
+
+ // query should be allowed only on non-faceted text fields
+ query_fields = {"cast"};
+ Option<nlohmann::json> result_op =
+ coll_mul_fields->search("anton", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false);
+ ASSERT_EQ(false, result_op.ok());
+ ASSERT_EQ(400, result_op.code());
+ ASSERT_EQ("Field `cast` is a faceted field - it cannot be used as a query field.", result_op.error());
+
+ // only faceted fields should be allowed for filtering
+ query_fields = {"title"};
+ result_op = coll_mul_fields->search("captain", query_fields, "starring: Samuel", facets, sort_fields, 0, 10, 1,
+ FREQUENCY, false);
+ ASSERT_EQ(false, result_op.ok());
+ ASSERT_EQ(400, result_op.code());
+ ASSERT_EQ("Field `starring` is not a faceted field - only a field marked as a facet can be filtered on.",
+ result_op.error());
+
+ collectionManager.drop_collection("coll_mul_fields");
+}
+
TEST_F(CollectionTest, FilterOnNumericFields) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
- field("years", field_types::INT32_ARRAY),
- field("timestamps", field_types::INT64_ARRAY)};
+ std::vector<field> fields = {
+ field("name", field_types::STRING, false),
+ field("age", field_types::INT32, false),
+ field("years", field_types::INT32_ARRAY, false),
+ field("timestamps", field_types::INT64_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, true)
+ };
+
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
- std::vector<field> sort_fields_index = { field("age", field_types::INT32) };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();
}
std::string json_line;
@@ -688,16 +740,18 @@ TEST_F(CollectionTest, FilterOnFloatFields) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
- field("top_3", field_types::FLOAT_ARRAY),
- field("rating", field_types::FLOAT)};
- std::vector<field> sort_fields_index = { field("rating", "FLOAT") };
+ std::vector<field> fields = {
+ field("name", field_types::STRING, false),
+ field("age", field_types::INT32, false),
+ field("top_3", field_types::FLOAT_ARRAY, false),
+ field("rating", field_types::FLOAT, false)
+ };
std::vector<sort_by> sort_fields_desc = { sort_by("rating", "DESC") };
std::vector<sort_by> sort_fields_asc = { sort_by("rating", "ASC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();
}
std::string json_line;
@@ -825,13 +879,17 @@ TEST_F(CollectionTest, SortOnFloatFields) {
Collection *coll_float_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/float_documents.jsonl");
- std::vector<field> fields = {field("title", field_types::STRING), field("score", field_types::FLOAT)};
- std::vector<field> sort_fields_index = { field("score", "FLOAT"), field("average", "FLOAT") };
+ std::vector<field> fields = {
+ field("title", field_types::STRING, false),
+ field("score", field_types::FLOAT, false),
+ field("average", field_types::FLOAT, false)
+ };
+
std::vector<sort_by> sort_fields_desc = { sort_by("score", "DESC"), sort_by("average", "DESC") };
coll_float_fields = collectionManager.get_collection("coll_float_fields");
if(coll_float_fields == nullptr) {
- coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, facet_fields, sort_fields_index).get();
+ coll_float_fields = collectionManager.create_collection("coll_float_fields", fields).get();
}
std::string json_line;
@@ -891,16 +949,18 @@ TEST_F(CollectionTest, FilterOnTextFields) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
- field("years", field_types::INT32_ARRAY),
- field("tags", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {
+ field("name", field_types::STRING, false),
+ field("age", field_types::INT32, false),
+ field("years", field_types::INT32_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, true)
+ };
- std::vector<field> sort_fields_index = { field("age", field_types::INT32) };
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();
}
std::string json_line;
@@ -972,17 +1032,16 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
- field("years", field_types::INT32_ARRAY),
- field("timestamps", field_types::INT64_ARRAY),
- field("tags", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {field("name", field_types::STRING, false), field("age", field_types::INT32, false),
+ field("years", field_types::INT32_ARRAY, false),
+ field("timestamps", field_types::INT64_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, false)};
- std::vector<field> sort_fields_index = { field("age", field_types::INT32) };
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();
}
std::string json_line;
@@ -1027,24 +1086,27 @@ TEST_F(CollectionTest, FacetCounts) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
- field("years", field_types::INT32_ARRAY),
- field("timestamps", field_types::INT64_ARRAY),
- field("tags", field_types::STRING_ARRAY)};
- facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("name_facet", field_types::STRING, true),
+ field("age", field_types::INT32, false),
+ field("years", field_types::INT32_ARRAY, false),
+ field("timestamps", field_types::INT64_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, true)};
- std::vector<field> sort_fields_index = { field("age", "DESC") };
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();
}
std::string json_line;
while (std::getline(infile, json_line)) {
- coll_array_fields->add(json_line);
+ nlohmann::json document = nlohmann::json::parse(json_line);
+ document["name_facet"] = document["name"];
+ const std::string & patched_json_line = document.dump();
+ coll_array_fields->add(patched_json_line);
}
infile.close();
@@ -1072,14 +1134,14 @@ TEST_F(CollectionTest, FacetCounts) {
// 2 facets, 1 text filter with no filters
facets.clear();
facets.push_back("tags");
- facets.push_back("name");
+ facets.push_back("name_facet");
results = coll_array_fields->search("Jeremy", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(2, results["facet_counts"].size());
ASSERT_EQ("tags", results["facet_counts"][0]["field_name"]);
- ASSERT_EQ("name", results["facet_counts"][1]["field_name"]);
+ ASSERT_EQ("name_facet", results["facet_counts"][1]["field_name"]);
// facet value must one that's stored, not indexed (i.e. no tokenization/standardization)
ASSERT_EQ("Jeremy Howard", results["facet_counts"][1]["counts"][0]["value"]);
@@ -1109,12 +1171,14 @@ TEST_F(CollectionTest, SortingOrder) {
Collection *coll_mul_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
- std::vector<field> fields = {field("title", field_types::STRING), field("starring", field_types::STRING),
- field("cast", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("starring", field_types::STRING, false),
+ field("points", field_types::INT32, false),
+ field("cast", field_types::STRING_ARRAY, false)};
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
- coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, facet_fields, sort_fields_index).get();
+ coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields).get();
}
std::string json_line;
@@ -1192,17 +1256,17 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
- std::vector<field> fields = {field("name", field_types::STRING), field("age", field_types::INT32),
- field("years", field_types::INT32_ARRAY),
- field("timestamps", field_types::INT64_ARRAY),
- field("tags", field_types::STRING_ARRAY)};
- facet_fields = {field("tags", field_types::STRING_ARRAY), field("name", field_types::STRING)};
- std::vector<field> sort_fields_index = { field("age", "DESC") };
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("age", field_types::INT32, false),
+ field("years", field_types::INT32_ARRAY, false),
+ field("timestamps", field_types::INT64_ARRAY, false),
+ field("tags", field_types::STRING_ARRAY, true)};
+
std::vector<sort_by> sort_fields = { sort_by("age", "DESC") };
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
- coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, facet_fields, sort_fields_index).get();
+ coll_array_fields = collectionManager.create_collection("coll_array_fields", fields).get();
}
std::string json_line;
@@ -1220,12 +1284,12 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
Option<nlohmann::json> res_op = coll_array_fields->search("the", query_fields_not_found, "", facets, sort_fields, 0, 10);
ASSERT_FALSE(res_op.ok());
ASSERT_EQ(400, res_op.code());
- ASSERT_STREQ("Could not find a search field named `titlez` in the schema.", res_op.error().c_str());
+ ASSERT_STREQ("Could not find a field named `titlez` in the schema.", res_op.error().c_str());
// when a query field is an integer field
res_op = coll_array_fields->search("the", {"age"}, "", facets, sort_fields, 0, 10);
ASSERT_EQ(400, res_op.code());
- ASSERT_STREQ("Search field `age` should be a string or a string array.", res_op.error().c_str());
+ ASSERT_STREQ("Field `age` should be a string or a string array.", res_op.error().c_str());
// when a facet field is not defined in the schema
res_op = coll_array_fields->search("the", {"name"}, "", {"timestamps"}, sort_fields, 0, 10);
@@ -1235,11 +1299,11 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
// when a rank field is not defined in the schema
res_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("timestamps", "ASC") }, 0, 10);
ASSERT_EQ(400, res_op.code());
- ASSERT_STREQ("Could not find a sort field named `timestamps` in the schema.", res_op.error().c_str());
+ ASSERT_STREQ("Could not find a field named `timestamps` in the schema.", res_op.error().c_str());
res_op = coll_array_fields->search("the", {"name"}, "", {}, { sort_by("_rank", "ASC") }, 0, 10);
ASSERT_EQ(400, res_op.code());
- ASSERT_STREQ("Could not find a sort field named `_rank` in the schema.", res_op.error().c_str());
+ ASSERT_STREQ("Could not find a field named `_rank` in the schema.", res_op.error().c_str());
collectionManager.drop_collection("coll_array_fields");
}
@@ -1248,45 +1312,45 @@ TEST_F(CollectionTest, IndexingWithBadData) {
// should not crash when document to-be-indexed doesn't match schema
Collection *sample_collection;
- std::vector<field> fields = {field("name", field_types::STRING)};
- facet_fields = {field("tags", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("tags", field_types::STRING_ARRAY, true),
+ field("age", field_types::INT32, false),
+ field("average", field_types::INT32, false) };
- std::vector<field> sort_fields_index = { field("age", field_types::INT32), field("average", field_types::INT32) };
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
sample_collection = collectionManager.get_collection("sample_collection");
if(sample_collection == nullptr) {
- sample_collection = collectionManager.create_collection("sample_collection", fields, facet_fields,
- sort_fields_index, "age").get();
+ sample_collection = collectionManager.create_collection("sample_collection", fields, "age").get();
}
- const Option<std::string> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29}");
+ const Option<std::string> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29, \"average\": 78}");
ASSERT_FALSE(search_fields_missing_op1.ok());
- ASSERT_STREQ("Field `name` has been declared as a search field in the schema, but is not found in the document.",
+ ASSERT_STREQ("Field `name` has been declared in the schema, but is not found in the document.",
search_fields_missing_op1.error().c_str());
- const Option<std::string> & search_fields_missing_op2 = sample_collection->add("{\"namez\": \"foo\", \"age\": 34}");
+ const Option<std::string> & search_fields_missing_op2 = sample_collection->add("{\"namez\": \"foo\", \"age\": 34, \"average\": 78}");
ASSERT_FALSE(search_fields_missing_op2.ok());
- ASSERT_STREQ("Field `name` has been declared as a search field in the schema, but is not found in the document.",
+ ASSERT_STREQ("Field `name` has been declared in the schema, but is not found in the document.",
search_fields_missing_op2.error().c_str());
- const Option<std::string> & facet_fields_missing_op1 = sample_collection->add("{\"name\": \"foo\", \"age\": 34}");
+ const Option<std::string> & facet_fields_missing_op1 = sample_collection->add("{\"name\": \"foo\", \"age\": 34, \"average\": 78}");
ASSERT_FALSE(facet_fields_missing_op1.ok());
- ASSERT_STREQ("Field `tags` has been declared as a facet field in the schema, but is not found in the document.",
+ ASSERT_STREQ("Field `tags` has been declared in the schema, but is not found in the document.",
facet_fields_missing_op1.error().c_str());
- const char *doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [\"red\", \"blue\"]}";
+ const char *doc_str = "{\"name\": \"foo\", \"age\": 34, \"avg\": 78, \"tags\": [\"red\", \"blue\"]}";
const Option<std::string> & sort_fields_missing_op1 = sample_collection->add(doc_str);
ASSERT_FALSE(sort_fields_missing_op1.ok());
- ASSERT_STREQ("Field `average` has been declared as a sort field in the schema, but is not found in the document.",
+ ASSERT_STREQ("Field `average` has been declared in the schema, but is not found in the document.",
sort_fields_missing_op1.error().c_str());
// Handle type errors
- doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22}";
+ doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": 22, \"average\": 78}";
const Option<std::string> & bad_facet_field_op = sample_collection->add(doc_str);
ASSERT_FALSE(bad_facet_field_op.ok());
- ASSERT_STREQ("Facet field `tags` must be a string array.", bad_facet_field_op.error().c_str());
+ ASSERT_STREQ("Field `tags` must be a string array.", bad_facet_field_op.error().c_str());
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": 34}";
const Option<std::string> & empty_facet_field_op = sample_collection->add(doc_str);
@@ -1311,7 +1375,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
doc_str = "{\"name\": \"foo\", \"age\": 34, \"tags\": [], \"average\": \"34\"}";
const Option<std::string> & bad_rank_field_op = sample_collection->add(doc_str);
ASSERT_FALSE(bad_rank_field_op.ok());
- ASSERT_STREQ("Sort field `average` must be a number.", bad_rank_field_op.error().c_str());
+ ASSERT_STREQ("Field `average` must be an int32.", bad_rank_field_op.error().c_str());
doc_str = "{\"name\": \"foo\", \"age\": asdadasd, \"tags\": [], \"average\": 34 }";
const Option<std::string> & bad_token_ranking_field_op4 = sample_collection->add(doc_str);
@@ -1324,15 +1388,16 @@ TEST_F(CollectionTest, IndexingWithBadData) {
TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
Collection *empty_coll;
- std::vector<field> fields = {field("name", field_types::STRING)};
- facet_fields = {field("tags", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("tags", field_types::STRING_ARRAY, false),
+ field("age", field_types::INT32, false),
+ field("average", field_types::INT32, false)};
- std::vector<field> sort_fields_index = { field("age", field_types::INT32), field("average", field_types::INT32) };
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
empty_coll = collectionManager.get_collection("empty_coll");
if(empty_coll == nullptr) {
- empty_coll = collectionManager.create_collection("empty_coll", fields, facet_fields, sort_fields_index, "age").get();
+ empty_coll = collectionManager.create_collection("empty_coll", fields, "age").get();
}
nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@@ -1343,15 +1408,16 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
TEST_F(CollectionTest, IdFieldShouldBeAString) {
Collection *coll1;
- std::vector<field> fields = {field("name", field_types::STRING)};
- facet_fields = {field("tags", field_types::STRING_ARRAY)};
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("tags", field_types::STRING_ARRAY, false),
+ field("age", field_types::INT32, false),
+ field("average", field_types::INT32, false)};
- std::vector<field> sort_fields_index = { field("age", field_types::INT32), field("average", field_types::INT32) };
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
- coll1 = collectionManager.create_collection("coll1", fields, facet_fields, sort_fields_index, "age").get();
+ coll1 = collectionManager.create_collection("coll1", fields, "age").get();
}
nlohmann::json doc;
@@ -1373,17 +1439,18 @@ TEST_F(CollectionTest, DeletionOfADocument) {
collectionManager.drop_collection("collection");
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
- std::vector<field> search_fields = {field("title", field_types::STRING)};
+
+ std::vector<field> search_fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false)};
+
+
std::vector<std::string> query_fields = {"title"};
- std::vector<field> facet_fields = { };
std::vector<sort_by> sort_fields = { sort_by("points", "DESC") };
- std::vector<field> sort_fields_index = { field("points", field_types::INT32) };
Collection *collection_for_del;
collection_for_del = collectionManager.get_collection("collection_for_del");
if(collection_for_del == nullptr) {
- collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, facet_fields,
- sort_fields_index, "points").get();
+ collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, "points").get();
}
std::string json_line;