Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow string fields to be filtered on.
Kishore Nallana2ce56fd675d
diff --git a/include/collection.h b/include/collection.hindex e499ef5c..038ce48e 100644--- a/include/collection.h+++ b/include/collection.h@@ -74,7 +74,7 @@ private:void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;- size_t union_of_leaf_ids(std::vector<const art_leaf *> &leaves, uint32_t **results_out);+ size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str);diff --git a/include/sorted_array.h b/include/sorted_array.hindex 93553cab..8f90b32e 100644--- a/include/sorted_array.h+++ b/include/sorted_array.h@@ -43,8 +43,4 @@ public:bool append(uint32_t value);void remove_values(uint32_t *sorted_values, uint32_t values_length);-- size_t intersect(uint32_t* arr, const size_t arr_length, uint32_t** results_out);-- size_t do_union(uint32_t* arr, const size_t arr_length, uint32_t** results_out);};\ No newline at end of filediff --git a/src/array_utils.cpp b/src/array_utils.cppindex ee458cf2..4459cac7 100644--- a/src/array_utils.cpp+++ b/src/array_utils.cpp@@ -47,6 +47,12 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,return lenB;}+ if(B == nullptr) {+ *out = new uint32_t[lenA];+ memcpy(*out, A, lenA * sizeof(uint32_t));+ return lenA;+ }+uint32_t* results = new uint32_t[lenA+lenB];while (indexA < lenA && indexB < lenB) {diff --git a/src/collection.cpp b/src/collection.cppindex 22d0bb2c..c92ffbcd 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -436,19 +436,20 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt}*/// initialize results with the starting element (for further intersection)- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();size_t result_size = query_suggestion[0]->values->ids.getLength();-if(result_size == 0) {continue;}+ uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();candidate_rank += 1;// intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)for(auto i=1; i < query_suggestion.size(); i++) {uint32_t* out = nullptr;- result_size = query_suggestion[i]->values->ids.intersect(result_ids, result_size, &out);+ uint32_t* ids = query_suggestion[i]->values->ids.uncompress();+ result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);+ delete[] ids;delete[] result_ids;result_ids = out;}@@ -500,16 +501,17 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt}}-size_t Collection::union_of_leaf_ids(std::vector<const art_leaf *> &leaves, uint32_t **results_out) {+size_t Collection::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,+ uint32_t **results_out) {uint32_t *results = nullptr;size_t results_length = 0;uint32_t *prev_results = nullptr;size_t prev_results_length = 0;- for(const art_leaf* leaf: leaves) {- results_length = leaf->values->ids.do_union(prev_results, prev_results_length, &results);-+ for(const std::pair<uint32_t*, size_t> & result_array_pair: result_array_pairs) {+ results_length = ArrayUtils::or_scalar(prev_results, prev_results_length, result_array_pair.first,+ result_array_pair.second, &results);delete [] prev_results;prev_results = results;prev_results_length = results_length;@@ -540,7 +542,7 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::}field _field = search_schema.at(field_name);- const std::string & raw_value = expression_parts[1];+ std::string & raw_value = expression_parts[1];filter f;if(_field.is_integer() || _field.is_float()) {@@ -590,10 +592,6 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::f = {field_name, {filter_value}, op_comparator.get()};}} else if(_field.is_string()) {- if(!_field.facet) {- return Option<>(400, "Field `" + _field.name + "` is not a faceted field - only a field marked as a "- "facet can be filtered on.");- }if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {std::vector<std::string> filter_values;StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");@@ -616,9 +614,11 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::if(search_index.count(a_filter.field_name) != 0) {art_tree* t = search_index.at(a_filter.field_name);field f = search_schema.at(a_filter.field_name);- std::vector<const art_leaf*> leaves;+ std::vector<std::pair<uint32_t*, size_t>> filter_result_array_pairs;if(f.is_integer()) {+ std::vector<const art_leaf*> leaves;+for(const std::string & filter_value: a_filter.values) {if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {int32_t value = (int32_t) std::stoi(filter_value);@@ -627,23 +627,62 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::int64_t value = (int64_t) std::stoi(filter_value);art_int64_search(t, value, a_filter.compare_operator, leaves);}++ for(const art_leaf* leaf: leaves) {+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),+ leaf->values->ids.getLength()));+ }}} else if(f.is_float()) {+ std::vector<const art_leaf*> leaves;+for(const std::string & filter_value: a_filter.values) {float value = (float) std::atof(filter_value.c_str());art_float_search(t, value, a_filter.compare_operator, leaves);+ for(const art_leaf* leaf: leaves) {+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),+ leaf->values->ids.getLength()));+ }}} else if(f.is_string()) {for(const std::string & filter_value: a_filter.values) {- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) filter_value.c_str(), filter_value.length()+1);- if(leaf != nullptr) {- leaves.push_back(leaf);+ // we have to tokenize the string, standardize it and then do an exact match+ std::vector<std::string> str_tokens;+ StringUtils::split(filter_value, str_tokens, " ");++ uint32_t* filtered_ids = nullptr;+ size_t filtered_size = 0;++ for(auto i = 0; i < str_tokens.size(); i++) {+ std::string & str_token = str_tokens[i];+ std::transform(str_token.begin(), str_token.end(), str_token.begin(), tolower);+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),+ str_token.length()+1);+ if(leaf == nullptr) {+ continue;+ }++ if(i == 0) {+ filtered_ids = leaf->values->ids.uncompress();+ filtered_size = leaf->values->ids.getLength();+ } else {+ // do AND for an exact match+ uint32_t* out = nullptr;+ uint32_t* leaf_ids = leaf->values->ids.uncompress();+ filtered_size = ArrayUtils::and_scalar(filtered_ids, filtered_size, leaf_ids,+ leaf->values->ids.getLength(), &out);+ delete[] leaf_ids;+ delete[] filtered_ids;+ filtered_ids = out;+ }}++ filter_result_array_pairs.push_back(std::make_pair(filtered_ids, filtered_size));}}uint32_t* result_ids = nullptr;- size_t result_ids_length = union_of_leaf_ids(leaves, &result_ids);+ size_t result_ids_length = union_of_ids(filter_result_array_pairs, &result_ids);if(filter_ids == nullptr) {filter_ids = result_ids;@@ -652,10 +691,14 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::uint32_t* filtered_results = nullptr;filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,result_ids_length, &filtered_results);- delete [] filter_ids;delete [] result_ids;+ delete [] filter_ids;filter_ids = filtered_results;}++ for(std::pair<uint32_t*, size_t> & filter_result_array_pair: filter_result_array_pairs) {+ delete[] filter_result_array_pair.first;+ }}}diff --git a/src/sorted_array.cpp b/src/sorted_array.cppindex 6ac6ae32..cffc553f 100644--- a/src/sorted_array.cpp+++ b/src/sorted_array.cpp@@ -171,67 +171,4 @@ void sorted_array::remove_values(uint32_t *sorted_values, uint32_t values_lengthload(new_array, new_index);delete[] curr_array;delete[] new_array;-}--size_t sorted_array::intersect(uint32_t* arr, const size_t arr_length, uint32_t** results_out) {- uint32_t* curr = uncompress();- uint32_t* results = nullptr;-- size_t results_length = ArrayUtils::and_scalar(arr, arr_length, curr, length, &results);- delete[] curr;-- *results_out = results;- return results_length;-}--size_t sorted_array::do_union(uint32_t *arr, const size_t arr_length, uint32_t** results_out) {- size_t curr_index = 0, arr_index = 0, res_index = 0;- uint32_t* curr = uncompress();- uint32_t* results = new uint32_t[arr_length+length];-- if(arr == nullptr) {- memcpy(results, curr, length * sizeof(uint32_t));- delete[] curr;- *results_out = results;- return length;- }-- while (curr_index < length && arr_index < arr_length) {- if (curr[curr_index] < arr[arr_index]) {- if(res_index == 0 || results[res_index-1] != curr[curr_index]) {- results[res_index] = curr[curr_index];- res_index++;- }- curr_index++;- } else {- if(res_index == 0 || results[res_index-1] != arr[arr_index]) {- results[res_index] = arr[arr_index];- res_index++;- }- arr_index++;- }- }-- while (arr_index < arr_length) {- if(results[res_index-1] != arr[arr_index]) {- results[res_index] = arr[arr_index];- res_index++;- }-- arr_index++;- }-- while (curr_index < length) {- if(results[res_index-1] != curr[curr_index]) {- results[res_index] = curr[curr_index];- res_index++;- }-- curr_index++;- }-- delete[] curr;-- *results_out = results;- return res_index;}\ No newline at end of filediff --git a/test/array_test.cpp b/test/array_test.cppindex d4eebd15..e04b41a4 100644--- a/test/array_test.cpp+++ b/test/array_test.cpp@@ -62,7 +62,7 @@ TEST(ArrayTest, Uncompress) {ASSERT_EQ(raw_unsorted_arr[i], unsorted.at(i));}- delete raw_unsorted_arr;+ delete[] raw_unsorted_arr;}TEST(ArrayTest, RemoveBetweenIndices) {diff --git a/test/array_utils_test.cpp b/test/array_utils_test.cppindex 5b64746f..018f613d 100644--- a/test/array_utils_test.cpp+++ b/test/array_utils_test.cpp@@ -92,4 +92,25 @@ TEST(SortedArrayTest, OrScalarMergeShouldRemoveDuplicatesAtBoundary) {delete[] results;delete[] arr1;delete[] arr2;+}++TEST(SortedArrayTest, OrScalarWithEitherArrayAsNull) {+ const size_t size1 = 9;+ uint32_t *arr1 = new uint32_t[size1];+ for(auto i = 0; i < 9; i++) {+ arr1[i] = i;+ }++ uint32_t *results = nullptr;+ uint32_t results_size = ArrayUtils::or_scalar(arr1, size1, nullptr, 0, &results);+ ASSERT_EQ(9, results_size);++ delete[] results;+ results = nullptr;++ results_size = ArrayUtils::or_scalar(nullptr, 0, arr1, size1, &results);+ ASSERT_EQ(9, results_size);++ delete[] results;+ results = nullptr;}\ No newline at end of filediff --git a/test/collection_test.cpp b/test/collection_test.cppindex 45819e99..3a4bb766 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -547,6 +547,18 @@ TEST_F(CollectionTest, MultipleFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}+ // filtering on unfaceted multi-valued string field+ query_fields = {"title"};+ results = coll_mul_fields->search("captain", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ids = {"6"};+ for(size_t i = 0; i < results["hits"].size(); i++) {+ nlohmann::json result = results["hits"].at(i);+ std::string result_id = result["id"];+ std::string id = ids.at(i);+ ASSERT_STREQ(id.c_str(), result_id.c_str());+ }+collectionManager.drop_collection("coll_mul_fields");}@@ -584,14 +596,13 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {ASSERT_EQ(400, result_op.code());ASSERT_EQ("Field `cast` is a faceted field - it cannot be used as a query field.", result_op.error());- // only faceted fields should be allowed for filtering+ // filtering on string field should be possiblequery_fields = {"title"};- result_op = coll_mul_fields->search("captain", query_fields, "starring: Samuel", facets, sort_fields, 0, 10, 1,+ result_op = coll_mul_fields->search("captain", query_fields, "starring: Samuel L. Jackson", facets, sort_fields, 0, 10, 1,FREQUENCY, false);- ASSERT_EQ(false, result_op.ok());- ASSERT_EQ(400, result_op.code());- ASSERT_EQ("Field `starring` is not a faceted field - only a field marked as a facet can be filtered on.",- result_op.error());+ ASSERT_EQ(true, result_op.ok());+ nlohmann::json results = result_op.get();+ ASSERT_EQ(1, results["hits"].size());collectionManager.drop_collection("coll_mul_fields");}@@ -1010,12 +1021,12 @@ TEST_F(CollectionTest, FilterOnTextFields) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- // should be exact matches (no normalization or fuzzy searching should happen)- results = coll_array_fields->search("Jeremy", query_fields, "tags: BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();- ASSERT_EQ(0, results["hits"].size());+ // need not be exact matches (normalization can happen)+ results = coll_array_fields->search("Jeremy", query_fields, "tags: BrONZe", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(2, results["hits"].size());// when comparators are used, should just treat them as part of search string- results = coll_array_fields->search("Jeremy", query_fields, "tags:<BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ results = coll_array_fields->search("Jeremy", query_fields, "tags:<bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(0, results["hits"].size());results = coll_array_fields->search("Jeremy", query_fields, "tags:<=BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cppindex 5dc14993..26bfa9c6 100644--- a/test/sorted_array_test.cpp+++ b/test/sorted_array_test.cpp@@ -67,7 +67,7 @@ TEST(SortedArrayTest, Uncompress) {ASSERT_EQ(raw_sorted_arr[i], sorted_arr.at(i));}- delete raw_sorted_arr;+ delete[] raw_sorted_arr;}TEST(SortedArrayTest, RemoveValues) {@@ -93,93 +93,6 @@ TEST(SortedArrayTest, RemoveValues) {}}-TEST(SortedArrayTest, Intersect) {- sorted_array arr1;- const size_t size1 = 9;- for(size_t i = 0; i < size1; i++) {- arr1.append(i);- }-- const size_t size2 = 10;- uint32_t *arr2 = new uint32_t[size2];- size_t arr2_len = 0;- for(size_t i = 2; i < size2; i++) {- if(i % 3 == 0) {- arr2[arr2_len++] = i;- }- }-- // arr1: [0..8] , arr2: [3, 6, 9]- uint32_t *results = nullptr;- uint32_t results_size = arr1.intersect(arr2, arr2_len, &results);- ASSERT_EQ(2, results_size);-- std::vector<uint32_t> expected = {3, 6};-- for(size_t i = 0; i < results_size; i++) {- ASSERT_EQ(expected[i], results[i]);- }-- delete [] results;-}--TEST(SortedArrayTest, MergeShouldRemoveDuplicates) {- sorted_array arr1;- const size_t size1 = 9;- for(size_t i = 0; i < size1; i++) {- arr1.append(i);- }-- const size_t size2 = 10;- uint32_t *arr2 = new uint32_t[size2];- size_t arr2_len = 0;- for(size_t i = 2; i < size2; i++) {- if(i % 3 == 0) {- arr2[arr2_len++] = i;- }- }-- // arr1: [0..8] , arr2: [3, 6, 9]- uint32_t *results = nullptr;- uint32_t results_size = arr1.do_union(arr2, arr2_len, &results);- ASSERT_EQ(10, results_size);-- std::vector<uint32_t> expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};-- for(size_t i = 0; i < results_size; i++) {- ASSERT_EQ(expected[i], results[i]);- }-- delete[] results;-}--TEST(SortedArrayTest, MergeShouldRemoveDuplicatesAtBoundary) {- std::vector<uint32_t> vec1 = {1, 2, 3, 5, 6, 7, 8};- sorted_array arr1;- for(auto i: vec1) {- arr1.append(i);- }-- std::vector<uint32_t> vec2 = {0, 4, 5};- uint32_t *arr2 = new uint32_t[vec2.size()];- auto j = 0;- for(auto i: vec2) {- arr2[j++] = i;- }-- uint32_t *results = nullptr;- uint32_t results_size = arr1.do_union(arr2, vec2.size(), &results);- ASSERT_EQ(9, results_size);-- std::vector<uint32_t> expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};-- for(size_t i = 0; i < results_size; i++) {- ASSERT_EQ(expected[i], results[i]);- }-- delete[] results;-}-TEST(SortedArrayTest, BulkIndexOf) {std::ifstream infile(std::string(ROOT_DIR)+"test/ids.txt");