Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `!=` compatibility for numeric type. (#835)
Harpreet Sangar34b16cdb21b6
diff --git a/.gitignore b/.gitignoreindex 9d0ed87d..2004a627 100644--- a/.gitignore+++ b/.gitignore@@ -11,3 +11,5 @@ cmake-build-release.DS_Store/bazel-*typesense-server-data/+.clwb/.bazelproject+.vscode/settings.json\ No newline at end of filediff --git a/.vscode/settings.json b/.vscode/settings.jsondeleted file mode 100644index b5dd0c6b..00000000--- a/.vscode/settings.json+++ /dev/null@@ -1,5 +0,0 @@-{- "clangd.arguments": [- "--compile-commands-dir=bazel-bin"- ]-}\ No newline at end of filediff --git a/include/field.h b/include/field.hindex d0ccbe80..9129af4b 100644--- a/include/field.h+++ b/include/field.h@@ -436,6 +436,10 @@ struct filter {std::string field_name;std::vector<std::string> values;std::vector<NUM_COMPARATOR> comparators;+ // Would be set when `field: != ...` is encountered with a string field or `field: != [ ... ]` is encountered in the+ // case of int and float fields. During filtering, all the results of matching the field against the values are+ // aggregated and then this flag is checked if negation on the aggregated result is required.+ bool apply_not_equals = false;static const std::string RANGE_OPERATOR() {return "..";@@ -473,6 +477,10 @@ struct filter {num_comparator = GREATER_THAN_EQUALS;}+ else if(comp_and_value.compare(0, 2, "!=") == 0) {+ num_comparator = NOT_EQUALS;+ }+else if(comp_and_value.compare(0, 1, "<") == 0) {num_comparator = LESS_THAN;}@@ -491,7 +499,7 @@ struct filter {if(num_comparator == LESS_THAN || num_comparator == GREATER_THAN) {comp_and_value = comp_and_value.substr(1);- } else if(num_comparator == LESS_THAN_EQUALS || num_comparator == GREATER_THAN_EQUALS) {+ } else if(num_comparator == LESS_THAN_EQUALS || num_comparator == GREATER_THAN_EQUALS || num_comparator == NOT_EQUALS) {comp_and_value = comp_and_value.substr(2);}diff --git a/include/index.h b/include/index.hindex 035998e8..e364fd70 100644--- a/include/index.h+++ b/include/index.h@@ -465,6 +465,11 @@ private:const size_t num_search_fields,std::vector<size_t>& popular_field_ids);+ void numeric_not_equals_filter(num_tree_t* const num_tree,+ const int64_t value,+ uint32_t*& ids,+ size_t& ids_len) const;+void do_filtering(uint32_t*& filter_ids,uint32_t& filter_ids_length,filter_node_t const* const root) const;diff --git a/src/field.cpp b/src/field.cppindex fb334dc4..c9df953f 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -122,6 +122,40 @@ Option<bool> toPostfix(std::queue<std::string>& tokens, std::queue<std::string>&return Option<bool>(true);}+Option<bool> toMultiValueNumericFilter(std::string& raw_value, filter& filter_exp, const field& _field) {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");+ filter_exp = {_field.name, {}, {}};+ for (std::string& filter_value: filter_values) {+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);+ if (!op_comparator.ok()) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());+ }+ if (op_comparator.get() == RANGE_INCLUSIVE) {+ // split the value around range operator to extract bounds+ std::vector<std::string> range_values;+ StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());+ for (const std::string& range_value: range_values) {+ auto validate_op = filter::validate_numerical_filter_value(_field, range_value);+ if (!validate_op.ok()) {+ return validate_op;+ }+ filter_exp.values.push_back(range_value);+ filter_exp.comparators.push_back(op_comparator.get());+ }+ } else {+ auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);+ if (!validate_op.ok()) {+ return validate_op;+ }+ filter_exp.values.push_back(filter_value);+ filter_exp.comparators.push_back(op_comparator.get());+ }+ }++ return Option<bool>(true);+}+Option<bool> toFilter(const std::string expression,filter& filter_exp,const tsl::htrie_map<char, field>& search_schema,@@ -204,34 +238,9 @@ Option<bool> toFilter(const std::string expression,if (_field.is_integer() || _field.is_float()) {// could be a single value or a listif (raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");- filter_exp = {field_name, {}, {}};- for (std::string& filter_value: filter_values) {- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(filter_value);- if (!op_comparator.ok()) {- return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());- }- if (op_comparator.get() == RANGE_INCLUSIVE) {- // split the value around range operator to extract bounds- std::vector<std::string> range_values;- StringUtils::split(filter_value, range_values, filter::RANGE_OPERATOR());- for (const std::string& range_value: range_values) {- auto validate_op = filter::validate_numerical_filter_value(_field, range_value);- if (!validate_op.ok()) {- return validate_op;- }- filter_exp.values.push_back(range_value);- filter_exp.comparators.push_back(op_comparator.get());- }- } else {- auto validate_op = filter::validate_numerical_filter_value(_field, filter_value);- if (!validate_op.ok()) {- return validate_op;- }- filter_exp.values.push_back(filter_value);- filter_exp.comparators.push_back(op_comparator.get());- }+ Option<bool> op = toMultiValueNumericFilter(raw_value, filter_exp, _field);+ if (!op.ok()) {+ return op;}} else {Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);@@ -251,6 +260,12 @@ Option<bool> toFilter(const std::string expression,filter_exp.values.push_back(range_value);filter_exp.comparators.push_back(op_comparator.get());}+ } else if (op_comparator.get() == NOT_EQUALS && raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ Option<bool> op = toMultiValueNumericFilter(raw_value, filter_exp, _field);+ if (!op.ok()) {+ return op;+ }+ filter_exp.apply_not_equals = true;} else {auto validate_op = filter::validate_numerical_filter_value(_field, raw_value);if (!validate_op.ok()) {@@ -353,6 +368,8 @@ Option<bool> toFilter(const std::string expression,} else {filter_exp = {field_name, {raw_value.substr(filter_value_index)}, {str_comparator}};}++ filter_exp.apply_not_equals = (str_comparator == NOT_EQUALS);} else {return Option<bool>(400, "Error with filter field `" + _field.name +"`: Unidentified field data type, see docs for supported data types.");diff --git a/src/index.cpp b/src/index.cppindex f85b55d3..6522fa62 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1548,6 +1548,35 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,}}+void Index::numeric_not_equals_filter(num_tree_t* const num_tree,+ const int64_t value,+ uint32_t*& ids,+ size_t& ids_len) const {+ uint32_t* to_exclude_ids = nullptr;+ size_t to_exclude_ids_len = 0;+ num_tree->search(EQUALS, value, &to_exclude_ids, to_exclude_ids_len);++ auto all_ids = seq_ids->uncompress();+ auto all_ids_size = seq_ids->num_ids();++ uint32_t* to_include_ids = nullptr;+ size_t to_include_ids_len = 0;++ to_include_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, to_exclude_ids,+ to_exclude_ids_len, &to_include_ids);++ delete[] all_ids;+ delete[] to_exclude_ids;++ uint32_t* out = nullptr;+ ids_len = ArrayUtils::or_scalar(ids, ids_len,+ to_include_ids, to_include_ids_len, &out);+ delete[] ids;+ delete[] to_include_ids;++ ids = out;+}+void Index::do_filtering(uint32_t*& filter_ids,uint32_t& filter_ids_length,filter_node_t const* const root) const {@@ -1603,6 +1632,8 @@ void Index::do_filtering(uint32_t*& filter_ids,int64_t range_end_value = (int64_t)std::stol(next_filter_value);num_tree->range_inclusive_search(value, range_end_value, &result_ids, result_ids_len);fi++;+ } else if (a_filter.comparators[fi] == NOT_EQUALS) {+ numeric_not_equals_filter(num_tree, value, result_ids, result_ids_len);} else {num_tree->search(a_filter.comparators[fi], value, &result_ids, result_ids_len);}@@ -1620,6 +1651,8 @@ void Index::do_filtering(uint32_t*& filter_ids,int64_t range_end_value = float_to_int64_t((float) std::atof(next_filter_value.c_str()));num_tree->range_inclusive_search(float_int64, range_end_value, &result_ids, result_ids_len);fi++;+ } else if (a_filter.comparators[fi] == NOT_EQUALS) {+ numeric_not_equals_filter(num_tree, value, result_ids, result_ids_len);} else {num_tree->search(a_filter.comparators[fi], float_int64, &result_ids, result_ids_len);}@@ -1631,28 +1664,7 @@ void Index::do_filtering(uint32_t*& filter_ids,for (const std::string& filter_value : a_filter.values) {int64_t bool_int64 = (filter_value == "1") ? 1 : 0;if (a_filter.comparators[value_index] == NOT_EQUALS) {- uint32_t* to_exclude_ids = nullptr;- size_t to_exclude_ids_len = 0;- num_tree->search(EQUALS, bool_int64, &to_exclude_ids, to_exclude_ids_len);-- auto all_ids = seq_ids->uncompress();- auto all_ids_size = seq_ids->num_ids();-- uint32_t* excluded_ids = nullptr;- size_t excluded_ids_len = 0;-- excluded_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, to_exclude_ids,- to_exclude_ids_len, &excluded_ids);-- delete[] all_ids;- delete[] to_exclude_ids;-- uint32_t* out = nullptr;- result_ids_len = ArrayUtils::or_scalar(result_ids, result_ids_len,- excluded_ids, excluded_ids_len, &out);- delete[] result_ids;- result_ids = out;- delete[] excluded_ids;+ numeric_not_equals_filter(num_tree, bool_int64, result_ids, result_ids_len);} else {num_tree->search(a_filter.comparators[value_index], bool_int64, &result_ids, result_ids_len);}@@ -1869,37 +1881,25 @@ void Index::do_filtering(uint32_t*& filter_ids,std::vector<uint32_t>().swap(f_id_buff); // clears out memory}- if (a_filter.comparators[0] == NOT_EQUALS) {- // exclude records from existing IDs (from previous filters or ALL records)- // "not equals" can only be applied to the entire array so we can do the exclusion operations once here- uint32_t* excluded_strt_ids = nullptr;- size_t excluded_strt_size = 0;+ result_ids = or_ids;+ result_ids_len = or_ids_size;+ }- if (result_ids == nullptr) {- if (filter_ids == nullptr) {- result_ids = seq_ids->uncompress();- result_ids_len = seq_ids->num_ids();- } else {- result_ids = filter_ids;- result_ids_len = filter_ids_length;- }- }+ if (a_filter.apply_not_equals) {+ auto all_ids = seq_ids->uncompress();+ auto all_ids_size = seq_ids->num_ids();- excluded_strt_size = ArrayUtils::exclude_scalar(result_ids, result_ids_len, or_ids,- or_ids_size, &excluded_strt_ids);+ uint32_t* to_include_ids = nullptr;+ size_t to_include_ids_len = 0;- if (filter_ids == nullptr) {- // means we had to uncompress `seq_ids` so need to free that- delete[] result_ids;- }+ to_include_ids_len = ArrayUtils::exclude_scalar(all_ids, all_ids_size, result_ids,+ result_ids_len, &to_include_ids);- delete[] or_ids;- or_ids = excluded_strt_ids;- or_ids_size = excluded_strt_size;- }+ delete[] all_ids;+ delete[] result_ids;- result_ids = or_ids;- result_ids_len = or_ids_size;+ result_ids = to_include_ids;+ result_ids_len = to_include_ids_len;}filter_ids = result_ids;diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 153fe084..39194688 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -598,6 +598,18 @@ TEST_F(CollectionFilteringTest, FilterOnNumericFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+ // not equals+ results = coll_array_fields->search("Jeremy", query_fields, "age:!= 24", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(4, results["hits"].size());++ ids = {"3", "1", "4", "2"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+// multiple filtersresults = coll_array_fields->search("Jeremy", query_fields, "years:<2005 && years:>1987", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());@@ -626,6 +638,42 @@ TEST_F(CollectionFilteringTest, FilterOnNumericFields) {results = coll_array_fields->search("Jeremy", query_fields, "age:= [21, 24, 63]", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(3, results["hits"].size());+ // individual comparators can still be applied.+ results = coll_array_fields->search("Jeremy", query_fields, "age: [!=21, >30]", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(4, results["hits"].size());++ ids = {"3", "1", "4", "0"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ // negate multiple search values (works like SQL's NOT IN) against a single int field+ results = coll_array_fields->search("Jeremy", query_fields, "age:!= [21, 24, 63]", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(2, results["hits"].size());++ ids = {"1", "4"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ // individual comparators can still be applied.+ results = coll_array_fields->search("Jeremy", query_fields, "age: != [<30, >60]", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(2, results["hits"].size());++ ids = {"1", "4"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }+// multiple search values against an int32 array field - also use extra padding between symbolsresults = coll_array_fields->search("Jeremy", query_fields, "years : [ 2015, 1985 , 1999]", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(4, results["hits"].size());@@ -753,6 +801,17 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {ASSERT_STREQ(id.c_str(), result_id.c_str()); //?}+ results = coll_array_fields->search("Jeremy", query_fields, "rating:!=0", facets, sort_fields_asc, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(4, results["hits"].size());++ ids = {"0", "4", "2", "1"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str()); //?+ }+// Searching on a float field, sorted desc by ratingresults = coll_array_fields->search("Jeremy", query_fields, "rating:>0.0", facets, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(4, results["hits"].size());@@ -802,6 +861,30 @@ TEST_F(CollectionFilteringTest, FilterOnFloatFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+ // negate multiple search values (works like SQL's NOT IN operator) against a single float field+ results = coll_array_fields->search("Jeremy", query_fields, "rating:!= [1.09, 7.812]", facets, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(3, results["hits"].size());++ ids = {"1", "4", "3"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }++ // individual comparators can still be applied.+ results = coll_array_fields->search("Jeremy", query_fields, "rating: != [<5.4, >9]", facets, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(2, results["hits"].size());++ ids = {"2", "4"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["document"]["id"];+ std::string id = ids.at(i);+ ASSERT_EQ(id, result_id);+ }+// multiple search values against a float array field - also use extra padding between symbolsresults = coll_array_fields->search("Jeremy", query_fields, "top_3 : [ 5.431, 0.001 , 7.812, 11.992]", facets, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(3, results["hits"].size());