Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow string fields to be filtered on.

Kishore Nallana2ce56fd675d
diff --git a/include/collection.h b/include/collection.h
index e499ef5c..038ce48e 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -74,7 +74,7 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
- size_t union_of_leaf_ids(std::vector<const art_leaf *> &leaves, uint32_t **results_out);
+ size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);
Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::string & simple_filter_str);
diff --git a/include/sorted_array.h b/include/sorted_array.h
index 93553cab..8f90b32e 100644
--- a/include/sorted_array.h
+++ b/include/sorted_array.h
@@ -43,8 +43,4 @@ public:
bool append(uint32_t value);
void remove_values(uint32_t *sorted_values, uint32_t values_length);
-
- size_t intersect(uint32_t* arr, const size_t arr_length, uint32_t** results_out);
-
- size_t do_union(uint32_t* arr, const size_t arr_length, uint32_t** results_out);
};
\ No newline at end of file
diff --git a/src/array_utils.cpp b/src/array_utils.cpp
index ee458cf2..4459cac7 100644
--- a/src/array_utils.cpp
+++ b/src/array_utils.cpp
@@ -47,6 +47,12 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,
return lenB;
}
+ if(B == nullptr) {
+ *out = new uint32_t[lenA];
+ memcpy(*out, A, lenA * sizeof(uint32_t));
+ return lenA;
+ }
+
uint32_t* results = new uint32_t[lenA+lenB];
while (indexA < lenA && indexB < lenB) {
diff --git a/src/collection.cpp b/src/collection.cpp
index 22d0bb2c..c92ffbcd 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -436,19 +436,20 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
}*/
// initialize results with the starting element (for further intersection)
- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
size_t result_size = query_suggestion[0]->values->ids.getLength();
-
if(result_size == 0) {
continue;
}
+ uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
candidate_rank += 1;
// intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
for(auto i=1; i < query_suggestion.size(); i++) {
uint32_t* out = nullptr;
- result_size = query_suggestion[i]->values->ids.intersect(result_ids, result_size, &out);
+ uint32_t* ids = query_suggestion[i]->values->ids.uncompress();
+ result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);
+ delete[] ids;
delete[] result_ids;
result_ids = out;
}
@@ -500,16 +501,17 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
}
}
-size_t Collection::union_of_leaf_ids(std::vector<const art_leaf *> &leaves, uint32_t **results_out) {
+size_t Collection::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,
+ uint32_t **results_out) {
uint32_t *results = nullptr;
size_t results_length = 0;
uint32_t *prev_results = nullptr;
size_t prev_results_length = 0;
- for(const art_leaf* leaf: leaves) {
- results_length = leaf->values->ids.do_union(prev_results, prev_results_length, &results);
-
+ for(const std::pair<uint32_t*, size_t> & result_array_pair: result_array_pairs) {
+ results_length = ArrayUtils::or_scalar(prev_results, prev_results_length, result_array_pair.first,
+ result_array_pair.second, &results);
delete [] prev_results;
prev_results = results;
prev_results_length = results_length;
@@ -540,7 +542,7 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
}
field _field = search_schema.at(field_name);
- const std::string & raw_value = expression_parts[1];
+ std::string & raw_value = expression_parts[1];
filter f;
if(_field.is_integer() || _field.is_float()) {
@@ -590,10 +592,6 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
f = {field_name, {filter_value}, op_comparator.get()};
}
} else if(_field.is_string()) {
- if(!_field.facet) {
- return Option<>(400, "Field `" + _field.name + "` is not a faceted field - only a field marked as a "
- "facet can be filtered on.");
- }
if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
std::vector<std::string> filter_values;
StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
@@ -616,9 +614,11 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
if(search_index.count(a_filter.field_name) != 0) {
art_tree* t = search_index.at(a_filter.field_name);
field f = search_schema.at(a_filter.field_name);
- std::vector<const art_leaf*> leaves;
+ std::vector<std::pair<uint32_t*, size_t>> filter_result_array_pairs;
if(f.is_integer()) {
+ std::vector<const art_leaf*> leaves;
+
for(const std::string & filter_value: a_filter.values) {
if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {
int32_t value = (int32_t) std::stoi(filter_value);
@@ -627,23 +627,62 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
int64_t value = (int64_t) std::stoi(filter_value);
art_int64_search(t, value, a_filter.compare_operator, leaves);
}
+
+ for(const art_leaf* leaf: leaves) {
+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
+ leaf->values->ids.getLength()));
+ }
}
} else if(f.is_float()) {
+ std::vector<const art_leaf*> leaves;
+
for(const std::string & filter_value: a_filter.values) {
float value = (float) std::atof(filter_value.c_str());
art_float_search(t, value, a_filter.compare_operator, leaves);
+ for(const art_leaf* leaf: leaves) {
+ filter_result_array_pairs.push_back(std::make_pair(leaf->values->ids.uncompress(),
+ leaf->values->ids.getLength()));
+ }
}
} else if(f.is_string()) {
for(const std::string & filter_value: a_filter.values) {
- art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) filter_value.c_str(), filter_value.length()+1);
- if(leaf != nullptr) {
- leaves.push_back(leaf);
+ // we have to tokenize the string, standardize it and then do an exact match
+ std::vector<std::string> str_tokens;
+ StringUtils::split(filter_value, str_tokens, " ");
+
+ uint32_t* filtered_ids = nullptr;
+ size_t filtered_size = 0;
+
+ for(auto i = 0; i < str_tokens.size(); i++) {
+ std::string & str_token = str_tokens[i];
+ std::transform(str_token.begin(), str_token.end(), str_token.begin(), tolower);
+ art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),
+ str_token.length()+1);
+ if(leaf == nullptr) {
+ continue;
+ }
+
+ if(i == 0) {
+ filtered_ids = leaf->values->ids.uncompress();
+ filtered_size = leaf->values->ids.getLength();
+ } else {
+ // do AND for an exact match
+ uint32_t* out = nullptr;
+ uint32_t* leaf_ids = leaf->values->ids.uncompress();
+ filtered_size = ArrayUtils::and_scalar(filtered_ids, filtered_size, leaf_ids,
+ leaf->values->ids.getLength(), &out);
+ delete[] leaf_ids;
+ delete[] filtered_ids;
+ filtered_ids = out;
+ }
}
+
+ filter_result_array_pairs.push_back(std::make_pair(filtered_ids, filtered_size));
}
}
uint32_t* result_ids = nullptr;
- size_t result_ids_length = union_of_leaf_ids(leaves, &result_ids);
+ size_t result_ids_length = union_of_ids(filter_result_array_pairs, &result_ids);
if(filter_ids == nullptr) {
filter_ids = result_ids;
@@ -652,10 +691,14 @@ Option<uint32_t> Collection::do_filtering(uint32_t** filter_ids_out, const std::
uint32_t* filtered_results = nullptr;
filter_ids_length = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
result_ids_length, &filtered_results);
- delete [] filter_ids;
delete [] result_ids;
+ delete [] filter_ids;
filter_ids = filtered_results;
}
+
+ for(std::pair<uint32_t*, size_t> & filter_result_array_pair: filter_result_array_pairs) {
+ delete[] filter_result_array_pair.first;
+ }
}
}
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
index 6ac6ae32..cffc553f 100644
--- a/src/sorted_array.cpp
+++ b/src/sorted_array.cpp
@@ -171,67 +171,4 @@ void sorted_array::remove_values(uint32_t *sorted_values, uint32_t values_length
load(new_array, new_index);
delete[] curr_array;
delete[] new_array;
-}
-
-size_t sorted_array::intersect(uint32_t* arr, const size_t arr_length, uint32_t** results_out) {
- uint32_t* curr = uncompress();
- uint32_t* results = nullptr;
-
- size_t results_length = ArrayUtils::and_scalar(arr, arr_length, curr, length, &results);
- delete[] curr;
-
- *results_out = results;
- return results_length;
-}
-
-size_t sorted_array::do_union(uint32_t *arr, const size_t arr_length, uint32_t** results_out) {
- size_t curr_index = 0, arr_index = 0, res_index = 0;
- uint32_t* curr = uncompress();
- uint32_t* results = new uint32_t[arr_length+length];
-
- if(arr == nullptr) {
- memcpy(results, curr, length * sizeof(uint32_t));
- delete[] curr;
- *results_out = results;
- return length;
- }
-
- while (curr_index < length && arr_index < arr_length) {
- if (curr[curr_index] < arr[arr_index]) {
- if(res_index == 0 || results[res_index-1] != curr[curr_index]) {
- results[res_index] = curr[curr_index];
- res_index++;
- }
- curr_index++;
- } else {
- if(res_index == 0 || results[res_index-1] != arr[arr_index]) {
- results[res_index] = arr[arr_index];
- res_index++;
- }
- arr_index++;
- }
- }
-
- while (arr_index < arr_length) {
- if(results[res_index-1] != arr[arr_index]) {
- results[res_index] = arr[arr_index];
- res_index++;
- }
-
- arr_index++;
- }
-
- while (curr_index < length) {
- if(results[res_index-1] != curr[curr_index]) {
- results[res_index] = curr[curr_index];
- res_index++;
- }
-
- curr_index++;
- }
-
- delete[] curr;
-
- *results_out = results;
- return res_index;
}
\ No newline at end of file
diff --git a/test/array_test.cpp b/test/array_test.cpp
index d4eebd15..e04b41a4 100644
--- a/test/array_test.cpp
+++ b/test/array_test.cpp
@@ -62,7 +62,7 @@ TEST(ArrayTest, Uncompress) {
ASSERT_EQ(raw_unsorted_arr[i], unsorted.at(i));
}
- delete raw_unsorted_arr;
+ delete[] raw_unsorted_arr;
}
TEST(ArrayTest, RemoveBetweenIndices) {
diff --git a/test/array_utils_test.cpp b/test/array_utils_test.cpp
index 5b64746f..018f613d 100644
--- a/test/array_utils_test.cpp
+++ b/test/array_utils_test.cpp
@@ -92,4 +92,25 @@ TEST(SortedArrayTest, OrScalarMergeShouldRemoveDuplicatesAtBoundary) {
delete[] results;
delete[] arr1;
delete[] arr2;
+}
+
+TEST(SortedArrayTest, OrScalarWithEitherArrayAsNull) {
+ const size_t size1 = 9;
+ uint32_t *arr1 = new uint32_t[size1];
+ for(auto i = 0; i < 9; i++) {
+ arr1[i] = i;
+ }
+
+ uint32_t *results = nullptr;
+ uint32_t results_size = ArrayUtils::or_scalar(arr1, size1, nullptr, 0, &results);
+ ASSERT_EQ(9, results_size);
+
+ delete[] results;
+ results = nullptr;
+
+ results_size = ArrayUtils::or_scalar(nullptr, 0, arr1, size1, &results);
+ ASSERT_EQ(9, results_size);
+
+ delete[] results;
+ results = nullptr;
}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 45819e99..3a4bb766 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -547,6 +547,18 @@ TEST_F(CollectionTest, MultipleFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
+ // filtering on unfaceted multi-valued string field
+ query_fields = {"title"};
+ results = coll_mul_fields->search("captain", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ids = {"6"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
collectionManager.drop_collection("coll_mul_fields");
}
@@ -584,14 +596,13 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
ASSERT_EQ(400, result_op.code());
ASSERT_EQ("Field `cast` is a faceted field - it cannot be used as a query field.", result_op.error());
- // only faceted fields should be allowed for filtering
+ // filtering on string field should be possible
query_fields = {"title"};
- result_op = coll_mul_fields->search("captain", query_fields, "starring: Samuel", facets, sort_fields, 0, 10, 1,
+ result_op = coll_mul_fields->search("captain", query_fields, "starring: Samuel L. Jackson", facets, sort_fields, 0, 10, 1,
FREQUENCY, false);
- ASSERT_EQ(false, result_op.ok());
- ASSERT_EQ(400, result_op.code());
- ASSERT_EQ("Field `starring` is not a faceted field - only a field marked as a facet can be filtered on.",
- result_op.error());
+ ASSERT_EQ(true, result_op.ok());
+ nlohmann::json results = result_op.get();
+ ASSERT_EQ(1, results["hits"].size());
collectionManager.drop_collection("coll_mul_fields");
}
@@ -1010,12 +1021,12 @@ TEST_F(CollectionTest, FilterOnTextFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- // should be exact matches (no normalization or fuzzy searching should happen)
- results = coll_array_fields->search("Jeremy", query_fields, "tags: BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
- ASSERT_EQ(0, results["hits"].size());
+ // need not be exact matches (normalization can happen)
+ results = coll_array_fields->search("Jeremy", query_fields, "tags: BrONZe", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ ASSERT_EQ(2, results["hits"].size());
// when comparators are used, should just treat them as part of search string
- results = coll_array_fields->search("Jeremy", query_fields, "tags:<BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
+ results = coll_array_fields->search("Jeremy", query_fields, "tags:<bronze", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
ASSERT_EQ(0, results["hits"].size());
results = coll_array_fields->search("Jeremy", query_fields, "tags:<=BRONZE", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();
diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cpp
index 5dc14993..26bfa9c6 100644
--- a/test/sorted_array_test.cpp
+++ b/test/sorted_array_test.cpp
@@ -67,7 +67,7 @@ TEST(SortedArrayTest, Uncompress) {
ASSERT_EQ(raw_sorted_arr[i], sorted_arr.at(i));
}
- delete raw_sorted_arr;
+ delete[] raw_sorted_arr;
}
TEST(SortedArrayTest, RemoveValues) {
@@ -93,93 +93,6 @@ TEST(SortedArrayTest, RemoveValues) {
}
}
-TEST(SortedArrayTest, Intersect) {
- sorted_array arr1;
- const size_t size1 = 9;
- for(size_t i = 0; i < size1; i++) {
- arr1.append(i);
- }
-
- const size_t size2 = 10;
- uint32_t *arr2 = new uint32_t[size2];
- size_t arr2_len = 0;
- for(size_t i = 2; i < size2; i++) {
- if(i % 3 == 0) {
- arr2[arr2_len++] = i;
- }
- }
-
- // arr1: [0..8] , arr2: [3, 6, 9]
- uint32_t *results = nullptr;
- uint32_t results_size = arr1.intersect(arr2, arr2_len, &results);
- ASSERT_EQ(2, results_size);
-
- std::vector<uint32_t> expected = {3, 6};
-
- for(size_t i = 0; i < results_size; i++) {
- ASSERT_EQ(expected[i], results[i]);
- }
-
- delete [] results;
-}
-
-TEST(SortedArrayTest, MergeShouldRemoveDuplicates) {
- sorted_array arr1;
- const size_t size1 = 9;
- for(size_t i = 0; i < size1; i++) {
- arr1.append(i);
- }
-
- const size_t size2 = 10;
- uint32_t *arr2 = new uint32_t[size2];
- size_t arr2_len = 0;
- for(size_t i = 2; i < size2; i++) {
- if(i % 3 == 0) {
- arr2[arr2_len++] = i;
- }
- }
-
- // arr1: [0..8] , arr2: [3, 6, 9]
- uint32_t *results = nullptr;
- uint32_t results_size = arr1.do_union(arr2, arr2_len, &results);
- ASSERT_EQ(10, results_size);
-
- std::vector<uint32_t> expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
-
- for(size_t i = 0; i < results_size; i++) {
- ASSERT_EQ(expected[i], results[i]);
- }
-
- delete[] results;
-}
-
-TEST(SortedArrayTest, MergeShouldRemoveDuplicatesAtBoundary) {
- std::vector<uint32_t> vec1 = {1, 2, 3, 5, 6, 7, 8};
- sorted_array arr1;
- for(auto i: vec1) {
- arr1.append(i);
- }
-
- std::vector<uint32_t> vec2 = {0, 4, 5};
- uint32_t *arr2 = new uint32_t[vec2.size()];
- auto j = 0;
- for(auto i: vec2) {
- arr2[j++] = i;
- }
-
- uint32_t *results = nullptr;
- uint32_t results_size = arr1.do_union(arr2, vec2.size(), &results);
- ASSERT_EQ(9, results_size);
-
- std::vector<uint32_t> expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};
-
- for(size_t i = 0; i < results_size; i++) {
- ASSERT_EQ(expected[i], results[i]);
- }
-
- delete[] results;
-}
-
TEST(SortedArrayTest, BulkIndexOf) {
std::ifstream infile(std::string(ROOT_DIR)+"test/ids.txt");