Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support indexing of bool fields.
Kishore Nallanfe0db598775a
diff --git a/TODO.md b/TODO.mdindex 804109c7..3b3ba8d9 100644--- a/TODO.md+++ b/TODO.md@@ -74,17 +74,19 @@- ~~Support token ranking field being a float~~- ~~https support~~- ~~Validate before string to int conversion in the http api layer~~+- ~~art bool support~~+- NOT operator support- handle hyphens (replace them)- get collection should show schema+- Iterator+- clean special chars before indexing+- > INT32_MAX validation for float field+- Proper logging- highlight of string arrays?-- NOT operator support- test for token ranking on float field- test for float int field deletion during doc deletion- Test for snippets- Test for replication-- Proper logging-- > INT32_MAX validation for float field-- art bool support- Add docs/explanation around ranking calc- Use rocksdb batch put for atomic insertion- Query token ids should match query token ordering@@ -94,9 +96,7 @@- Delete using range: https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys- Test for string utils- Prevent string copy during indexing-- clean special chars before indexing- Minimum results should be a variable instead of blindly going with max_results-- Iterator- UTF-8 support for fuzzy search- Handle searching for non-existing fields gracefully- test for same match score but different primary, secondary attrdiff --git a/include/field.h b/include/field.hindex da0745ce..aebd90e8 100644--- a/include/field.h+++ b/include/field.h@@ -10,10 +10,12 @@ namespace field_types {static const std::string INT32 = "int32";static const std::string INT64 = "int64";static const std::string FLOAT = "float";- static const std::string FLOAT_ARRAY = "float[]";+ static const std::string BOOL = "bool";static const std::string STRING_ARRAY = "string[]";static const std::string INT32_ARRAY = "int32[]";static const std::string INT64_ARRAY = "int64[]";+ static const std::string FLOAT_ARRAY = "float[]";+ static const std::string BOOL_ARRAY = "bool[]";}namespace fields {@@ -39,6 +41,10 @@ struct field {return (type == field_types::FLOAT);}+ bool is_single_bool() const {+ return (type == field_types::BOOL);+ }+bool is_integer() const {return (type == field_types::INT32 || type == field_types::INT32_ARRAY ||type == field_types::INT64 || type == field_types::INT64_ARRAY);@@ -48,6 +54,10 @@ struct field {return (type == field_types::FLOAT || type == field_types::FLOAT_ARRAY);}+ bool is_bool() const {+ return (type == field_types::BOOL || type == field_types::BOOL_ARRAY);+ }+bool is_string() const {return (type == field_types::STRING || type == field_types::STRING_ARRAY);}diff --git a/include/index.h b/include/index.hindex 101b9aee..e51b52ad 100644--- a/include/index.h+++ b/include/index.h@@ -22,7 +22,7 @@ struct token_candidates {struct search_args {std::string query;std::vector<std::string> search_fields;- std::string simple_filter_query;+ std::vector<filter> filters;std::vector<facet> facets;std::vector<sort_by> sort_fields_std;int num_typos;@@ -33,17 +33,18 @@ struct search_args {std::vector<std::pair<int, Topster<512>::KV>> field_order_kvs;size_t all_result_ids_len;std::vector<std::vector<art_leaf*>> searched_queries;+ Option<uint32_t> outcome;- search_args() {+ search_args(): outcome(0) {}- search_args(std::string query, std::vector<std::string> search_fields, std::string simple_filter_query,+ search_args(std::string query, std::vector<std::string> search_fields, std::vector<filter> filters,std::vector<facet> facets, std::vector<sort_by> sort_fields_std, int num_typos,size_t per_page, size_t page, token_ordering token_order, bool prefix):- query(query), search_fields(search_fields), simple_filter_query(simple_filter_query), facets(facets),+ query(query), search_fields(search_fields), filters(filters), facets(facets),sort_fields_std(sort_fields_std), num_typos(num_typos), per_page(per_page), page(page),- token_order(token_order), prefix(prefix), all_result_ids_len(0) {+ token_order(token_order), prefix(prefix), all_result_ids_len(0), outcome(0) {}};@@ -75,7 +76,7 @@ private:size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str);+ Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);void do_facets(std::vector<facet> & facets, uint32_t* result_ids, size_t results_size);@@ -107,12 +108,16 @@ private:void index_int64_field(const int64_t value, const uint32_t score, art_tree *t, uint32_t seq_id) const;void index_float_field(const float value, const uint32_t score, art_tree *t, uint32_t seq_id) const;++ void index_bool_field(const bool value, const uint32_t score, art_tree *t, uint32_t seq_id) const;void index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;void index_int64_array_field(const std::vector<int64_t> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;void index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;++ void index_bool_array_field(const std::vector<bool> & values, const uint32_t score, art_tree *t, uint32_t seq_id) const;void remove_and_shift_offset_index(sorted_array &offset_index, const uint32_t *indices_sorted,const uint32_t indices_length);@@ -127,8 +132,8 @@ public:void run_search();- Option<size_t> search(std::string query, const std::vector<std::string> search_fields,- const std::string & simple_filter_query, std::vector<facet> & facets,+ void search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,+ const std::vector<filter> & filters, std::vector<facet> & facets,std::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page,const token_ordering token_order, const bool prefix,diff --git a/src/collection.cpp b/src/collection.cppindex 62751a18..19e2e3f3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -20,7 +20,7 @@ Collection::Collection(const std::string name, const uint32_t collection_id, confacet_schema.emplace(field.name, field);}- if(field.is_single_integer() || field.is_single_float()) {+ if(field.is_single_integer() || field.is_single_float() || field.is_single_bool()) {sort_schema.emplace(field.name, field);}}@@ -142,6 +142,10 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docuif(!document[field_name].is_number()) { // allows integer to be passed to a float fieldreturn Option<>(400, "Field `" + field_name + "` must be a float.");}+ } else if(field_pair.second.type == field_types::BOOL) {+ if(!document[field_name].is_boolean()) {+ return Option<>(400, "Field `" + field_name + "` must be a bool.");+ }} else if(field_pair.second.type == field_types::STRING_ARRAY) {if(!document[field_name].is_array()) {return Option<>(400, "Field `" + field_name + "` must be a string array.");@@ -173,6 +177,14 @@ Option<uint32_t> Collection::validate_index_in_memory(const nlohmann::json &docuif(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) {return Option<>(400, "Field `" + field_name + "` must be a float array.");}+ } else if(field_pair.second.type == field_types::BOOL_ARRAY) {+ if(!document[field_name].is_array()) {+ return Option<>(400, "Field `" + field_name + "` must be a bool array.");+ }++ if(document[field_name].size() > 0 && !document[field_name][0].is_boolean()) {+ return Option<>(400, "Field `" + field_name + "` must be a bool array.");+ }}}@@ -258,6 +270,95 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s}}+ // validate filter fields+ std::vector<std::string> filter_blocks;+ StringUtils::split(simple_filter_query, filter_blocks, "&&");++ std::vector<filter> filters;+ for(const std::string & filter_block: filter_blocks) {+ // split into [field_name, value]+ std::vector<std::string> expression_parts;+ StringUtils::split(filter_block, expression_parts, ":");+ if(expression_parts.size() != 2) {+ return Option<nlohmann::json>(400, "Could not parse the filter query.");+ }++ const std::string & field_name = expression_parts[0];+ if(search_schema.count(field_name) == 0) {+ return Option<nlohmann::json>(400, "Could not find a filter field named `" + field_name + "` in the schema.");+ }++ field _field = search_schema.at(field_name);+ std::string & raw_value = expression_parts[1];+ filter f;++ if(_field.is_integer() || _field.is_float()) {+ // could be a single value or a list+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");++ for(const std::string & filter_value: filter_values) {+ if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not an integer.");+ }++ if(_field.is_float() && !StringUtils::is_float(filter_value)) {+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not a float.");+ }+ }++ f = {field_name, filter_values, EQUALS};+ } else {+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);+ if(!op_comparator.ok()) {+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: " + op_comparator.error());+ }++ // extract numerical value+ std::string filter_value;+ if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {+ filter_value = raw_value.substr(1);+ } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {+ filter_value = raw_value.substr(2);+ } else {+ // EQUALS+ filter_value = raw_value;+ }++ filter_value = StringUtils::trim(filter_value);++ if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not an integer.");+ }++ if(_field.is_float() && !StringUtils::is_float(filter_value)) {+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Not a float.");+ }++ f = {field_name, {filter_value}, op_comparator.get()};+ }+ } else if(_field.is_bool()) {+ if(raw_value != "true" && raw_value != "false") {+ return Option<nlohmann::json>(400, "Value of field `" + _field.name + "`: must be `true` or `false`.");+ }+ std::string bool_value = (raw_value == "true") ? "1" : "0";+ f = {field_name, {bool_value}, EQUALS};+ } else if(_field.is_string()) {+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");+ f = {field_name, filter_values, EQUALS};+ } else {+ f = {field_name, {raw_value}, EQUALS};+ }+ } else {+ return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Unidentified field type.");+ }++ filters.push_back(f);+ }+// validate facet fieldsfor(const std::string & field_name: facet_fields) {if(facet_schema.count(field_name) == 0) {@@ -308,7 +409,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s// send data to individual index threadsfor(Index* index: indices) {- index->search_params = search_args(query, search_fields, simple_filter_query, facets, sort_fields_std,+ index->search_params = search_args(query, search_fields, filters, facets, sort_fields_std,num_typos, per_page, page, token_order, prefix);{std::lock_guard<std::mutex> lk(index->m);@@ -318,6 +419,8 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sindex->cv.notify_one();}+ Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads+for(Index* index: indices) {// wait for the worker{@@ -325,6 +428,16 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sindex->cv.wait(lk, [index]{return index->processed;});}+ if(!index->search_params.outcome.ok()) {+ index_search_op = Option<nlohmann::json>(index->search_params.outcome.code(),+ index->search_params.outcome.error());+ }++ if(!index_search_op.ok()) {+ // we still need to iterate without breaking to release the locks+ continue;+ }+// need to remap the search query index before appendingfor(auto & field_order_kv: index->search_params.field_order_kvs) {field_order_kv.second.query_index += searched_queries.size();@@ -355,6 +468,10 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<stotal_found += index->search_params.all_result_ids_len;}+ if(!index_search_op.ok()) {+ return index_search_op;+ }+// All fields are sorted descendingstd::sort(field_order_kvs.begin(), field_order_kvs.end(),[](const std::pair<int, Topster<512>::KV> & a, const std::pair<int, Topster<512>::KV> & b) {diff --git a/src/index.cpp b/src/index.cppindex 1a8ab447..1cee91d2 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -69,6 +69,9 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t} else if(field_pair.second.type == field_types::FLOAT) {float value = document[field_name];index_float_field(value, points, t, seq_id);+ } else if(field_pair.second.type == field_types::BOOL) {+ bool value = document[field_name];+ index_bool_field(value, points, t, seq_id);} else if(field_pair.second.type == field_types::STRING_ARRAY) {std::vector<std::string> strings = document[field_name];index_string_array_field(strings, points, t, seq_id, field_pair.second.is_facet());@@ -81,17 +84,22 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t} else if(field_pair.second.type == field_types::FLOAT_ARRAY) {std::vector<float> values = document[field_name];index_float_array_field(values, points, t, seq_id);+ } else if(field_pair.second.type == field_types::BOOL_ARRAY) {+ std::vector<bool> values = document[field_name];+ index_bool_array_field(values, points, t, seq_id);}// add numerical values automatically into sort indexif(field_pair.second.type == field_types::INT32 || field_pair.second.type == field_types::INT64 ||- field_pair.second.type == field_types::FLOAT) {+ field_pair.second.type == field_types::FLOAT || field_pair.second.type == field_types::BOOL) {spp::sparse_hash_map<uint32_t, number_t> *doc_to_score = sort_index.at(field_pair.first);if(document[field_pair.first].is_number_integer()) {doc_to_score->emplace(seq_id, document[field_pair.first].get<int64_t>());} else if(document[field_pair.first].is_number_float()) {doc_to_score->emplace(seq_id, document[field_pair.first].get<float>());+ } else if(document[field_pair.first].is_boolean()) {+ doc_to_score->emplace(seq_id, (int64_t) document[field_pair.first].get<bool>());}}}@@ -158,6 +166,29 @@ void Index::index_int64_field(const int64_t value, uint32_t score, art_tree *t,art_insert(t, key, KEY_LEN, &art_doc, num_hits);}+void Index::index_bool_field(const bool value, const uint32_t score, art_tree *t, uint32_t seq_id) const {+ const int KEY_LEN = 1;+ unsigned char key[KEY_LEN];+ key[0] = value ? '1' : '0';+ //key[1] = '\0';++ uint32_t num_hits = 0;+ art_leaf* leaf = (art_leaf *) art_search(t, key, KEY_LEN);+ if(leaf != NULL) {+ num_hits = leaf->values->ids.getLength();+ }++ num_hits += 1;++ art_document art_doc;+ art_doc.id = seq_id;+ art_doc.score = score;+ art_doc.offsets_len = 0;+ art_doc.offsets = nullptr;++ art_insert(t, key, KEY_LEN, &art_doc, num_hits);+}+void Index::index_float_field(const float value, uint32_t score, art_tree *t, uint32_t seq_id) const {const int KEY_LEN = 8;unsigned char key[KEY_LEN];@@ -249,6 +280,13 @@ void Index::index_int64_array_field(const std::vector<int64_t> & values, const u}}+void Index::index_bool_array_field(const std::vector<bool> & values, const uint32_t score, art_tree *t,+ uint32_t seq_id) const {+ for(const bool value: values) {+ index_bool_field(value, score, t, seq_id);+ }+}+void Index::index_float_array_field(const std::vector<float> & values, const uint32_t score, art_tree *t,uint32_t seq_id) const {for(const float value: values) {@@ -382,95 +420,10 @@ size_t Index::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_arreturn results_length;}-Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str) {- // parse the filter string- std::vector<std::string> filter_blocks;- StringUtils::split(simple_filter_str, filter_blocks, "&&");-- std::vector<filter> filters;-- for(const std::string & filter_block: filter_blocks) {- // split into [field_name, value]- std::vector<std::string> expression_parts;- StringUtils::split(filter_block, expression_parts, ":");- if(expression_parts.size() != 2) {- return Option<>(400, "Could not parse the filter query.");- }-- const std::string & field_name = expression_parts[0];- if(search_schema.count(field_name) == 0) {- return Option<>(400, "Could not find a filter field named `" + field_name + "` in the schema.");- }-- field _field = search_schema.at(field_name);- std::string & raw_value = expression_parts[1];- filter f;-- if(_field.is_integer() || _field.is_float()) {- // could be a single value or a list- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");-- for(const std::string & filter_value: filter_values) {- if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {- return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");- }-- if(_field.is_float() && !StringUtils::is_float(filter_value)) {- return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");- }- }-- f = {field_name, filter_values, EQUALS};- } else {- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);- if(!op_comparator.ok()) {- return Option<>(400, "Error with field `" + _field.name + "`: " + op_comparator.error());- }-- // extract numerical value- std::string filter_value;- if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {- filter_value = raw_value.substr(1);- } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {- filter_value = raw_value.substr(2);- } else {- // EQUALS- filter_value = raw_value;- }-- filter_value = StringUtils::trim(filter_value);-- if(_field.is_integer() && !StringUtils::is_integer(filter_value)) {- return Option<>(400, "Error with field `" + _field.name + "`: Not an integer.");- }-- if(_field.is_float() && !StringUtils::is_float(filter_value)) {- return Option<>(400, "Error with field `" + _field.name + "`: Not a float.");- }-- f = {field_name, {filter_value}, op_comparator.get()};- }- } else if(_field.is_string()) {- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");- f = {field_name, filter_values, EQUALS};- } else {- f = {field_name, {raw_value}, EQUALS};- }- } else {- return Option<>(400, "Error with field `" + _field.name + "`: Unidentified field type.");- }-- filters.push_back(f);- }-+Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {uint32_t* filter_ids = nullptr;uint32_t filter_ids_length = 0;- // process the filters firstfor(auto i = 0; i < filters.size(); i++) {const filter & a_filter = filters[i];@@ -507,6 +460,17 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::strinleaf->values->ids.getLength()));}}+ } else if(f.is_bool()) {+ std::vector<const art_leaf*> leaves;++ for(const std::string & filter_value: a_filter.values) {+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) filter_value.c_str(),+ filter_value.length());+ if(leaf) {+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),+ leaf->values->ids.getLength()));+ }+ }} else if(f.is_string()) {for(const std::string & filter_value: a_filter.values) {// we have to tokenize the string, standardize it and then do an exact match@@ -580,7 +544,8 @@ void Index::run_search() {}// after the wait, we own the lock.- search(search_params.query, search_params.search_fields, search_params.simple_filter_query, search_params.facets,+ search(search_params.outcome, search_params.query, search_params.search_fields,+ search_params.filters, search_params.facets,search_params.sort_fields_std, search_params.num_typos, search_params.per_page, search_params.page,search_params.token_order, search_params.prefix, search_params.field_order_kvs,search_params.all_result_ids_len, search_params.searched_queries);@@ -595,8 +560,8 @@ void Index::run_search() {}}-Option<size_t> Index::search(std::string query, const std::vector<std::string> search_fields,- const std::string & simple_filter_query, std::vector<facet> & facets,+void Index::search(Option<uint32_t> & outcome, std::string query, const std::vector<std::string> search_fields,+ const std::vector<filter> & filters, std::vector<facet> & facets,std::vector<sort_by> sort_fields_std, const int num_typos,const size_t per_page, const size_t page, const token_ordering token_order,const bool prefix, std::vector<std::pair<int, Topster<512>::KV>> & field_order_kvs,@@ -607,9 +572,10 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> s// process the filters firstuint32_t* filter_ids = nullptr;- Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, simple_filter_query);+ Option<uint32_t> op_filter_ids_length = do_filtering(&filter_ids, filters);if(!op_filter_ids_length.ok()) {- return Option<size_t>(op_filter_ids_length.code(), op_filter_ids_length.error());+ outcome = Option<uint32_t>(op_filter_ids_length);+ return ;}const uint32_t filter_ids_length = op_filter_ids_length.get();@@ -622,7 +588,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> sTopster<512> topster;const std::string & field = search_fields[i];// proceed to query search only when no filters are provided or when filtering produces results- if(simple_filter_query.size() == 0 || filter_ids_length > 0) {+ if(filters.size() == 0 || filter_ids_length > 0) {search_field(query, field, filter_ids, filter_ids_length, facets, sort_fields_std, num_typos, num_results,searched_queries, topster, &all_result_ids, all_result_ids_len, token_order, prefix);topster.sort();@@ -640,7 +606,7 @@ Option<size_t> Index::search(std::string query, const std::vector<std::string> slong long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;- return Option<size_t>(field_order_kvs.size());+ outcome = Option<uint32_t>(field_order_kvs.size());}/*@@ -1053,6 +1019,20 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)encode_float(value, key);tokens.push_back(std::string((char*)key, KEY_LEN));}+ } else if(name_field.second.type == field_types::BOOL) {+ const int KEY_LEN = 1;+ unsigned char key[KEY_LEN];+ bool value = document[name_field.first].get<bool>();+ key[0] = value ? '1' : '0';+ tokens.push_back(std::string((char*)key, KEY_LEN));+ } else if(name_field.second.type == field_types::BOOL_ARRAY) {+ std::vector<bool> values = document[name_field.first].get<std::vector<bool>>();+ for(const bool value: values) {+ const int KEY_LEN = 1;+ unsigned char key[KEY_LEN];+ key[0] = value ? '1' : '0';+ tokens.push_back(std::string((char*)key, KEY_LEN));+ }}for(auto & token: tokens) {diff --git a/test/bool_documents.jsonl b/test/bool_documents.jsonlnew file mode 100644index 00000000..6a6f74ae--- /dev/null+++ b/test/bool_documents.jsonl@@ -0,0 +1,10 @@+{"popular": true, "title": "Shawshank Redemption", "rating": 9.9, "bool_array": [true, true]}+{"popular": true, "title": "The Godfather", "rating": 9.9, "bool_array": [false, true]}+{"popular": false, "title": "Daniel the Wizard", "rating": 1.6, "bool_array": [true, true]}+{"popular": true, "title": "The Schindler's List", "rating": 9.8, "bool_array": [false, false]}+{"popular": true, "title": "The Wizard of Oz", "rating": 8.9, "bool_array": [true, true]}+{"popular": true, "title": "Casablanca", "rating": 9.1, "bool_array": [true, false]}+{"popular": true, "title": "Raging Bull", "rating": 9.0, "bool_array": [true, true]}+{"popular": true, "title": "Citizen Kane", "rating": 8.9, "bool_array": [false, false]}+{"popular": false, "title": "Saving Christmas", "rating": 1.5, "bool_array": [false, true]}+{"popular": false, "title": "The Legend of the Titanic", "rating": 2.0, "bool_array": [true, false]}\ No newline at end of filediff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex adac570a..3d36c785 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -19,7 +19,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, "auth_key");+ collectionManager.init(store, "auth_key", "search_auth_key");search_fields = {field("title", field_types::STRING, false),@@ -123,7 +123,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {// create a new collection manager to ensure that it restores the records from the disk backed storeCollectionManager & collectionManager2 = CollectionManager::get_instance();- collectionManager2.init(store, "auth_key");+ collectionManager2.init(store, "auth_key", "search_auth_key");collection1 = collectionManager2.get_collection("collection1");ASSERT_NE(nullptr, collection1);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex def6c454..b4a6c717 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -22,7 +22,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, "auth_key");+ collectionManager.init(store, "auth_key", "search_auth_key");std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");std::vector<field> search_fields = {@@ -956,6 +956,93 @@ TEST_F(CollectionTest, SortOnFloatFields) {collectionManager.drop_collection("coll_float_fields");}+TEST_F(CollectionTest, QueryBoolFields) {+ Collection *coll_bool;++ std::ifstream infile(std::string(ROOT_DIR)+"test/bool_documents.jsonl");+ std::vector<field> fields = {+ field("popular", field_types::BOOL, false),+ field("title", field_types::STRING, false),+ field("rating", field_types::FLOAT, false),+ field("bool_array", field_types::BOOL_ARRAY, false),+ };++ std::vector<sort_by> sort_fields = { sort_by("popular", "DESC"), sort_by("rating", "DESC") };++ coll_bool = collectionManager.get_collection("coll_bool");+ if(coll_bool == nullptr) {+ coll_bool = collectionManager.create_collection("coll_bool", fields).get();+ }++ std::string json_line;++ while (std::getline(infile, json_line)) {+ Option<std::string> op = coll_bool->add(json_line);+ }++ infile.close();++ // Plain search with no filters - results should be sorted correctly+ query_fields = {"title"};+ std::vector<std::string> facets;+ nlohmann::json results = coll_bool->search("the", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(5, results["hits"].size());++ std::vector<std::string> ids = {"1", "3", "4", "9", "2"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // Searching on a bool field+ results = coll_bool->search("the", query_fields, "popular:true", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(3, results["hits"].size());++ ids = {"1", "3", "4"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ results = coll_bool->search("the", query_fields, "popular:false", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(2, results["hits"].size());++ ids = {"9", "2"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ // searching against a bool array field++ // should be able to search only with a single boolean value+ Option<nlohmann::json> res_op = coll_bool->search("the", query_fields, "bool_array:[true, false]", facets,+ sort_fields, 0, 10, 1, FREQUENCY, false);+ ASSERT_EQ(false, res_op.ok());++ results = coll_bool->search("the", query_fields, "bool_array: true", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(4, results["hits"].size());+ ids = {"1", "4", "9", "2"};++ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }++ collectionManager.drop_collection("coll_bool");+}+TEST_F(CollectionTest, FilterOnTextFields) {Collection *coll_array_fields;@@ -1082,9 +1169,9 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {results = coll_array_fields->search("Jeremy", query_fields, "timestamps abcdef", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(0, results["hits"].size());- // just empty spaces+ // just spaces - must be treated as empty filterresults = coll_array_fields->search("Jeremy", query_fields, " ", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(5, results["hits"].size());// wrapping number with quotesresults = coll_array_fields->search("Jeremy", query_fields, "age: '21'", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();