Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support indexing of bool fields.

Kishore Nallanfe0db598775a
diff --git a/TODO.md b/TODO.md
index 804109c7..3b3ba8d9 100644
--- a/TODO.md
+++ b/TODO.md
@@ -74,17 +74,19 @@
- ~~Support token ranking field being a float~~
- ~~https support~~
- ~~Validate before string to int conversion in the http api layer~~
+- ~~art bool support~~
+- NOT operator support
- handle hyphens (replace them)
- get collection should show schema
+- Iterator
+- clean special chars before indexing
+- > INT32_MAX validation for float field
+- Proper logging
- highlight of string arrays?
-- NOT operator support
- test for token ranking on float field
- test for float int field deletion during doc deletion
- Test for snippets
- Test for replication
-- Proper logging
-- > INT32_MAX validation for float field
-- art bool support
- Add docs/explanation around ranking calc
- Use rocksdb batch put for atomic insertion
- Query token ids should match query token ordering
@@ -94,9 +96,7 @@
- Delete using range: https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys
- Test for string utils
- Prevent string copy during indexing
-- clean special chars before indexing
- Minimum results should be a variable instead of blindly going with max_results
-- Iterator
- UTF-8 support for fuzzy search
- Handle searching for non-existing fields gracefully
- test for same match score but different primary, secondary attr
diff --git a/include/field.h b/include/field.h
index da0745ce..aebd90e8 100644
--- a/include/field.h
+++ b/include/field.h
@@ -10,10 +10,12 @@ namespace field_types {
static const std::string INT32 = "int32";
static const std::string INT64 = "int64";
static const std::string FLOAT = "float";
- static const std::string FLOAT_ARRAY = "float[]";
+ static const std::string BOOL = "bool";
static const std::string STRING_ARRAY = "string[]";
static const std::string INT32_ARRAY = "int32[]";
static const std::string INT64_ARRAY = "int64[]";
+ static const std::string FLOAT_ARRAY = "float[]";
+ static const std::string BOOL_ARRAY = "bool[]";
}
namespace fields {
@@ -39,6 +41,10 @@ struct field {
return (type == field_types::FLOAT);
}
+ bool is_single_bool() const {
+ return (type == field_types::BOOL);
+ }
+
bool is_integer() const {
return (type == field_types::INT32 || type == field_types::INT32_ARRAY ||
type == field_types::INT64 || type == field_types::INT64_ARRAY);
@@ -48,6 +54,10 @@ struct field {
return (type == field_types::FLOAT || type == field_types::FLOAT_ARRAY);
}
+ bool is_bool() const {
+ return (type == field_types::BOOL || type == field_types::BOOL_ARRAY);
+ }
+
bool is_string() const {
return (type == field_types::STRING || type == field_types::STRING_ARRAY);
}
diff --git a/include/index.h b/include/index.h
index 101b9aee..e51b52ad 100644
--- a/include/index.h
+++ b/include/index.h
@@ -22,7 +22,7 @@ struct token_candidates {
struct search_args {
std::string query;
std::vector<std::string> search_fields;
- std::string simple_filter_query;
+ std::vector<filter> filters;
std::vector<facet> facets;
std::vector<sort_by> sort_fields_std;
int num_typos;
@@ -33,17 +33,18 @@ struct search_args {
std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;
size_t all_result_ids_len;
std::vector<std::vector<art_leaf*>> searched_queries;
+ Option<uint32_t> outcome;
- search_args() {
+ search_args(): outcome(0) {
}
- search_args(std::string query, std::vector<std::string> search_fields, std::string simple_filter_query,
+ search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,
std::vector<facet> facets, std::vector<sort_by> sort_fields_std, int num_typos,
size_t per_page, size_t page, token_ordering token_order, bool prefix):
- query(query), search_fields(search_fields), simple_filter_query(simple_filter_query), facets(facets),
+ query(query), search_fields(search_fields), filters(filters), facets(facets),
sort_fields_std(sort_fields_std), num_typos(num_typos), per_page(per_page), page(page),
- token_order(token_order), prefix(prefix), all_result_ids_len(0) {
+ token_order(token_order), prefix(prefix), all_result_ids_len(0), outcome(0) {
}
};
@@ -75,7 +76,7 @@ private:
size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);
- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str);
+ Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);
@@ -107,12 +108,16 @@ private:
void index_int64_field(const int64_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
void index_float_field(const float value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_bool_field(const bool value, const uint32_t score, art_tree *t, uint32_t seq_id) const;
void index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
void index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
+
+ void index_bool_array_field(const std::vector<bool> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;
void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,
const uint32_t indices_length);
@@ -127,8 +132,8 @@ public:
void run_search();
- Option<size_t> search(std::string query, const std::vector<std::string> search_fields,
- const std::string & simple_filter_query, std::vector<facet> & facets,
+ void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,
+ const std::vector<filter> & filters, std::vector<facet> & facets,
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page,
const token_ordering token_order, const bool prefix,
diff --git a/src/collection.cpp b/src/collection.cpp
index 62751a18..19e2e3f3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -20,7 +20,7 @@ Collection::Collection(const std::string name, const uint32_t collection_id, con
facet_schema.emplace(field.name, field);
}
- if(field.is_single_integer() || field.is_single_float()) {
+ if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {
sort_schema.emplace(field.name, field);
}
}
@@ -142,6 +142,10 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docu
if(!document[field_name].is_number()) { // allows integer to be passed to a float field
return Option<>(400, "Field `" + field_name + "` must be a float.");
}
+ } else if(field_pair.second.type == field_types::BOOL) {
+ if(!document[field_name].is_boolean()) {
+ return Option<>(400, "Field `" + field_name + "` must be a bool.");
+ }
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
if(!document[field_name].is_array()) {
return Option<>(400, "Field `" + field_name + "` must be a string array.");
@@ -173,6 +177,14 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docu
if(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) {
return Option<>(400, "Field `" + field_name + "` must be a float array.");
}
+ } else if(field_pair.second.type == field_types::BOOL_ARRAY) {
+ if(!document[field_name].is_array()) {
+ return Option<>(400, "Field `" + field_name + "` must be a bool array.");
+ }
+
+ if(document[field_name].size() > 0 && !document[field_name][0].is_boolean()) {
+ return Option<>(400, "Field `" + field_name + "` must be a bool array.");
+ }
}
}
@@ -258,6 +270,95 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
}
+ // validate filter fields
+ std::vector<std::string> filter_blocks;
+ StringUtils::split(simple_filter_query, filter_blocks, "&&");
+
+ std::vector<filter> filters;
+ for(const std::string & filter_block: filter_blocks) {
+ // split into [field_name, value]
+ std::vector<std::string> expression_parts;
+ StringUtils::split(filter_block, expression_parts, ":");
+ if(expression_parts.size() != 2) {
+ return Option<nlohmann::json>(400, "Could not parse the filter query.");
+ }
+
+ const std::string & field_name = expression_parts[0];
+ if(search_schema.count(field_name) == 0) {
+ return Option<nlohmann::json>(400, "Could not find a filter field named `" + field_name + "` in the schema.");
+ }
+
+ field _field = search_schema.at(field_name);
+ std::string & raw_value = expression_parts[1];
+ filter f;
+
+ if(_field.is_integer() || _field.is_float()) {
+ // could be a single value or a list
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+
+ for(const std::string & filter_value: filter_values) {
+ if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not an integer.");
+ }
+
+ if(_field.is_float() && !StringUtils::is_float(filter_value)) {
+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not a float.");
+ }
+ }
+
+ f = {field_name, filter_values, EQUALS};
+ } else {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
+ if(!op_comparator.ok()) {
+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: " + op_comparator.error());
+ }
+
+ // extract numerical value
+ std::string filter_value;
+ if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {
+ filter_value = raw_value.substr(1);
+ } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {
+ filter_value = raw_value.substr(2);
+ } else {
+ // EQUALS
+ filter_value = raw_value;
+ }
+
+ filter_value = StringUtils::trim(filter_value);
+
+ if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not an integer.");
+ }
+
+ if(_field.is_float() && !StringUtils::is_float(filter_value)) {
+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not a float.");
+ }
+
+ f = {field_name, {filter_value}, op_comparator.get()};
+ }
+ } else if(_field.is_bool()) {
+ if(raw_value != "true" && raw_value != "false") {
+ return Option<nlohmann::json>(400, "Value of field `" + _field.name + "`: must be `true` or `false`.");
+ }
+ std::string bool_value = (raw_value == "true") ? "1" : "0";
+ f = {field_name, {bool_value}, EQUALS};
+ } else if(_field.is_string()) {
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+ f = {field_name, filter_values, EQUALS};
+ } else {
+ f = {field_name, {raw_value}, EQUALS};
+ }
+ } else {
+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Unidentified field type.");
+ }
+
+ filters.push_back(f);
+ }
+
// validate facet fields
for(const std::string & field_name: facet_fields) {
if(facet_schema.count(field_name) == 0) {
@@ -308,7 +409,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
// send data to individual index threads
for(Index* index: indices) {
- index->search_params = search_args(query, search_fields, simple_filter_query, facets, sort_fields_std,
+ index->search_params = search_args(query, search_fields, filters, facets, sort_fields_std,
num_typos, per_page, page, token_order, prefix);
{
std::lock_guard<std::mutex> lk(index->m);
@@ -318,6 +419,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
index->cv.notify_one();
}
+ Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads
+
for(Index* index: indices) {
// wait for the worker
{
@@ -325,6 +428,16 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
index->cv.wait(lk, [index]{return index->processed;});
}
+ if(!index->search_params.outcome.ok()) {
+ index_search_op = Option<nlohmann::json>(index->search_params.outcome.code(),
+ index->search_params.outcome.error());
+ }
+
+ if(!index_search_op.ok()) {
+ // we still need to iterate without breaking to release the locks
+ continue;
+ }
+
// need to remap the search query index before appending
for(auto & field_order_kv: index->search_params.field_order_kvs) {
field_order_kv.second.query_index += searched_queries.size();
@@ -355,6 +468,10 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
total_found += index->search_params.all_result_ids_len;
}
+ if(!index_search_op.ok()) {
+ return index_search_op;
+ }
+
// All fields are sorted descending
std::sort(field_order_kvs.begin(), field_order_kvs.end(),
[](const std::pair<int, Topster<512>::KV> & a, const std::pair<int, Topster<512>::KV> & b) {
diff --git a/src/index.cpp b/src/index.cpp
index 1a8ab447..1cee91d2 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -69,6 +69,9 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
} else if(field_pair.second.type == field_types::FLOAT) {
float value = document[field_name];
index_float_field(value, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::BOOL) {
+ bool value = document[field_name];
+ index_bool_field(value, points, t, seq_id);
} else if(field_pair.second.type == field_types::STRING_ARRAY) {
std::vector<std::string> strings = document[field_name];
index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());
@@ -81,17 +84,22 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t
} else if(field_pair.second.type == field_types::FLOAT_ARRAY) {
std::vector<float> values = document[field_name];
index_float_array_field(values, points, t, seq_id);
+ } else if(field_pair.second.type == field_types::BOOL_ARRAY) {
+ std::vector<bool> values = document[field_name];
+ index_bool_array_field(values, points, t, seq_id);
}
// add numerical values automatically into sort index
if(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||
- field_pair.second.type == field_types::FLOAT) {
+ field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL) {
spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);
if(document[field_pair.first].is_number_integer()) {
doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());
} else if(document[field_pair.first].is_number_float()) {
doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());
+ } else if(document[field_pair.first].is_boolean()) {
+ doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());
}
}
}
@@ -158,6 +166,29 @@ void Index::index_int64_field(const int64_t value, uint32_t score, art_tree *t,
art_insert(t, key, KEY_LEN, &art_doc, num_hits);
}
+void Index::index_bool_field(const bool value, const uint32_t score, art_tree *t, uint32_t seq_id) const {
+ const int KEY_LEN = 1;
+ unsigned char key[KEY_LEN];
+ key[0] = value ? '1' : '0';
+ //key[1] = '\0';
+
+ uint32_t num_hits = 0;
+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);
+ if(leaf != NULL) {
+ num_hits = leaf->values->ids.getLength();
+ }
+
+ num_hits += 1;
+
+ art_document art_doc;
+ art_doc.id = seq_id;
+ art_doc.score = score;
+ art_doc.offsets_len = 0;
+ art_doc.offsets = nullptr;
+
+ art_insert(t, key, KEY_LEN, &art_doc, num_hits);
+}
+
void Index::index_float_field(const float value, uint32_t score, art_tree *t, uint32_t seq_id) const {
const int KEY_LEN = 8;
unsigned char key[KEY_LEN];
@@ -249,6 +280,13 @@ void Index::index_int64_array_field(const std::vector<int64_t> & values, const u
}
}
+void Index::index_bool_array_field(const std::vector<bool> & values, const uint32_t score, art_tree *t,
+ uint32_t seq_id) const {
+ for(const bool value: values) {
+ index_bool_field(value, score, t, seq_id);
+ }
+}
+
void Index::index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t,
uint32_t seq_id) const {
for(const float value: values) {
@@ -382,95 +420,10 @@ size_t Index::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_ar
return results_length;
}
-Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str) {
- // parse the filter string
- std::vector<std::string> filter_blocks;
- StringUtils::split(simple_filter_str, filter_blocks, "&&");
-
- std::vector<filter> filters;
-
- for(const std::string & filter_block: filter_blocks) {
- // split into [field_name, value]
- std::vector<std::string> expression_parts;
- StringUtils::split(filter_block, expression_parts, ":");
- if(expression_parts.size() != 2) {
- return Option<>(400, "Could not parse the filter query.");
- }
-
- const std::string & field_name = expression_parts[0];
- if(search_schema.count(field_name) == 0) {
- return Option<>(400, "Could not find a filter field named `" + field_name + "` in the schema.");
- }
-
- field _field = search_schema.at(field_name);
- std::string & raw_value = expression_parts[1];
- filter f;
-
- if(_field.is_integer() || _field.is_float()) {
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
-
- for(const std::string & filter_value: filter_values) {
- if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");
- }
-
- if(_field.is_float() && !StringUtils::is_float(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");
- }
- }
-
- f = {field_name, filter_values, EQUALS};
- } else {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
- if(!op_comparator.ok()) {
- return Option<>(400, "Error with field `" + _field.name + "`: " + op_comparator.error());
- }
-
- // extract numerical value
- std::string filter_value;
- if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {
- filter_value = raw_value.substr(1);
- } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {
- filter_value = raw_value.substr(2);
- } else {
- // EQUALS
- filter_value = raw_value;
- }
-
- filter_value = StringUtils::trim(filter_value);
-
- if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");
- }
-
- if(_field.is_float() && !StringUtils::is_float(filter_value)) {
- return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");
- }
-
- f = {field_name, {filter_value}, op_comparator.get()};
- }
- } else if(_field.is_string()) {
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
- f = {field_name, filter_values, EQUALS};
- } else {
- f = {field_name, {raw_value}, EQUALS};
- }
- } else {
- return Option<>(400, "Error with field `" + _field.name + "`: Unidentified field type.");
- }
-
- filters.push_back(f);
- }
-
+Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_length = 0;
- // process the filters first
for(auto i = 0; i < filters.size(); i++) {
const filter & a_filter = filters[i];
@@ -507,6 +460,17 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::strin
leaf->values->ids.getLength()));
}
}
+ } else if(f.is_bool()) {
+ std::vector<const art_leaf*> leaves;
+
+ for(const std::string & filter_value: a_filter.values) {
+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) filter_value.c_str(),
+ filter_value.length());
+ if(leaf) {
+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
+ leaf->values->ids.getLength()));
+ }
+ }
} else if(f.is_string()) {
for(const std::string & filter_value: a_filter.values) {
// we have to tokenize the string, standardize it and then do an exact match
@@ -580,7 +544,8 @@ void Index::run_search() {
}
// after the wait, we own the lock.
- search(search_params.query, search_params.search_fields, search_params.simple_filter_query, search_params.facets,
+ search(search_params.outcome, search_params.query, search_params.search_fields,
+ search_params.filters, search_params.facets,
search_params.sort_fields_std, search_params.num_typos, search_params.per_page, search_params.page,
search_params.token_order, search_params.prefix, search_params.field_order_kvs,
search_params.all_result_ids_len, search_params.searched_queries);
@@ -595,8 +560,8 @@ void Index::run_search() {
}
}
-Option<size_t> Index::search(std::string query, const std::vector<std::string> search_fields,
- const std::string & simple_filter_query, std::vector<facet> & facets,
+void Index::search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,
+ const std::vector<filter> & filters, std::vector<facet> & facets,
std::vector<sort_by> sort_fields_std, const int num_typos,
const size_t per_page, const size_t page, const token_ordering token_order,
const bool prefix, std::vector<std::pair<int, Topster<512>::KV>> & field_order_kvs,
@@ -607,9 +572,10 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s
// process the filters first
uint32_t* filter_ids = nullptr;
- Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, simple_filter_query);
+ Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, filters);
if(!op_filter_ids_length.ok()) {
- return Option<size_t>(op_filter_ids_length.code(), op_filter_ids_length.error());
+ outcome = Option<uint32_t>(op_filter_ids_length);
+ return ;
}
const uint32_t filter_ids_length = op_filter_ids_length.get();
@@ -622,7 +588,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s
Topster<512> topster;
const std::string & field = search_fields[i];
// proceed to query search only when no filters are provided or when filtering produces results
- if(simple_filter_query.size() == 0 || filter_ids_length > 0) {
+ if(filters.size() == 0 || filter_ids_length > 0) {
search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,
searched_queries, topster, &all_result_ids, all_result_ids_len, token_order, prefix);
topster.sort();
@@ -640,7 +606,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
- return Option<size_t>(field_order_kvs.size());
+ outcome = Option<uint32_t>(field_order_kvs.size());
}
/*
@@ -1053,6 +1019,20 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)
encode_float(value, key);
tokens.push_back(std::string((char*)key, KEY_LEN));
}
+ } else if(name_field.second.type == field_types::BOOL) {
+ const int KEY_LEN = 1;
+ unsigned char key[KEY_LEN];
+ bool value = document[name_field.first].get<bool>();
+ key[0] = value ? '1' : '0';
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ } else if(name_field.second.type == field_types::BOOL_ARRAY) {
+ std::vector<bool> values = document[name_field.first].get<std::vector<bool>>();
+ for(const bool value: values) {
+ const int KEY_LEN = 1;
+ unsigned char key[KEY_LEN];
+ key[0] = value ? '1' : '0';
+ tokens.push_back(std::string((char*)key, KEY_LEN));
+ }
}
for(auto & token: tokens) {
diff --git a/test/bool_documents.jsonl b/test/bool_documents.jsonl
new file mode 100644
index 00000000..6a6f74ae
--- /dev/null
+++ b/test/bool_documents.jsonl
@@ -0,0 +1,10 @@
+{"popular": true, "title": "Shawshank Redemption", "rating": 9.9, "bool_array": [true, true]}
+{"popular": true, "title": "The Godfather", "rating": 9.9, "bool_array": [false, true]}
+{"popular": false, "title": "Daniel the Wizard", "rating": 1.6, "bool_array": [true, true]}
+{"popular": true, "title": "The Schindler's List", "rating": 9.8, "bool_array": [false, false]}
+{"popular": true, "title": "The Wizard of Oz", "rating": 8.9, "bool_array": [true, true]}
+{"popular": true, "title": "Casablanca", "rating": 9.1, "bool_array": [true, false]}
+{"popular": true, "title": "Raging Bull", "rating": 9.0, "bool_array": [true, true]}
+{"popular": true, "title": "Citizen Kane", "rating": 8.9, "bool_array": [false, false]}
+{"popular": false, "title": "Saving Christmas", "rating": 1.5, "bool_array": [false, true]}
+{"popular": false, "title": "The Legend of the Titanic", "rating": 2.0, "bool_array": [true, false]}
\ No newline at end of file
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index adac570a..3d36c785 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -19,7 +19,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, "auth_key");
+ collectionManager.init(store, "auth_key", "search_auth_key");
search_fields = {
field("title", field_types::STRING, false),
@@ -123,7 +123,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
- collectionManager2.init(store, "auth_key");
+ collectionManager2.init(store, "auth_key", "search_auth_key");
collection1 = collectionManager2.get_collection("collection1");
ASSERT_NE(nullptr, collection1);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index def6c454..b4a6c717 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -22,7 +22,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, "auth_key");
+ collectionManager.init(store, "auth_key", "search_auth_key");
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
std::vector<field> search_fields = {
@@ -956,6 +956,93 @@ TEST_F(CollectionTest, SortOnFloatFields) {
collectionManager.drop_collection("coll_float_fields");
}
+TEST_F(CollectionTest, QueryBoolFields) {
+ Collection *coll_bool;
+
+ std::ifstream infile(std::string(ROOT_DIR)+"test/bool_documents.jsonl");
+ std::vector<field> fields = {
+ field("popular", field_types::BOOL, false),
+ field("title", field_types::STRING, false),
+ field("rating", field_types::FLOAT, false),
+ field("bool_array", field_types::BOOL_ARRAY, false),
+ };
+
+ std::vector<sort_by> sort_fields = { sort_by("popular", "DESC"), sort_by("rating", "DESC") };
+
+ coll_bool = collectionManager.get_collection("coll_bool");
+ if(coll_bool == nullptr) {
+ coll_bool = collectionManager.create_collection("coll_bool", fields).get();
+ }
+
+ std::string json_line;
+
+ while (std::getline(infile, json_line)) {
+ Option<std::string> op = coll_bool->add(json_line);
+ }
+
+ infile.close();
+
+ // Plain search with no filters - results should be sorted correctly
+ query_fields = {"title"};
+ std::vector<std::string> facets;
+ nlohmann::json results = coll_bool->search("the", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(5, results["hits"].size());
+
+ std::vector<std::string> ids = {"1", "3", "4", "9", "2"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // Searching on a bool field
+ results = coll_bool->search("the", query_fields, "popular:true", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(3, results["hits"].size());
+
+ ids = {"1", "3", "4"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ results = coll_bool->search("the", query_fields, "popular:false", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(2, results["hits"].size());
+
+ ids = {"9", "2"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // searching against a bool array field
+
+ // should be able to search only with a single boolean value
+ Option<nlohmann::json> res_op = coll_bool->search("the", query_fields, "bool_array:[true, false]", facets,
+ sort_fields, 0, 10, 1, FREQUENCY, false);
+ ASSERT_EQ(false, res_op.ok());
+
+ results = coll_bool->search("the", query_fields, "bool_array: true", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(4, results["hits"].size());
+ ids = {"1", "4", "9", "2"};
+
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ collectionManager.drop_collection("coll_bool");
+}
+
TEST_F(CollectionTest, FilterOnTextFields) {
Collection *coll_array_fields;
@@ -1082,9 +1169,9 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
results = coll_array_fields->search("Jeremy", query_fields, "timestamps abcdef", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(0, results["hits"].size());
- // just empty spaces
+ // just spaces - must be treated as empty filter
results = coll_array_fields->search("Jeremy", query_fields, " ", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(0, results["hits"].size());
+ ASSERT_EQ(5, results["hits"].size());
// wrapping number with quotes
results = coll_array_fields->search("Jeremy", query_fields, "age: '21'", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();