Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support for "IN" style numerical filter.

Kishore Nallan14168c48fc44
diff --git a/include/collection.h b/include/collection.h
index fdc39b92..9ac8d11d 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -39,7 +39,7 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
- void union_with_filter_ids(std::vector<const art_leaf*> & leaves, uint32_t** filter_ids, uint32_t & filter_ids_length);
+ size_t union_of_leaf_ids(std::vector<const art_leaf *> &leaves, uint32_t **results_out);
void search(uint32_t* filter_ids, size_t filter_ids_length, std::string & query, const std::string & field,
const int num_typos, const size_t num_results, Topster<100> & topster, size_t & num_found,
diff --git a/include/field.h b/include/field.h
index b83b933d..8a2efe18 100644
--- a/include/field.h
+++ b/include/field.h
@@ -28,7 +28,7 @@ struct field {
struct filter {
std::string field_name;
- std::string value_json;
+ std::vector<std::string> values;
std::string compare_operator;
NUM_COMPARATOR get_comparator() const {
diff --git a/include/sorted_array.h b/include/sorted_array.h
index 099b32b0..8798e0b6 100644
--- a/include/sorted_array.h
+++ b/include/sorted_array.h
@@ -44,7 +44,7 @@ public:
void remove_values(uint32_t *sorted_values, uint32_t values_length);
- size_t intersect(uint32_t* arr, const size_t arr_length, uint32_t* results);
+ size_t intersect(uint32_t* arr, const size_t arr_length, uint32_t** results_out);
- size_t do_union(uint32_t* arr, const size_t arr_length, uint32_t* results);
+ size_t do_union(uint32_t* arr, const size_t arr_length, uint32_t** results_out);
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index c8c311e4..88150246 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -215,8 +215,8 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
// intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
for(auto i=1; i < query_suggestion.size(); i++) {
- uint32_t* out = new uint32_t[std::min(result_size, (size_t) query_suggestion[i]->values->ids.getLength())];
- result_size = query_suggestion[i]->values->ids.intersect(result_ids, result_size, out);
+ uint32_t* out = nullptr;
+ result_size = query_suggestion[i]->values->ids.intersect(result_ids, result_size, &out);
delete[] result_ids;
result_ids = out;
}
@@ -247,18 +247,23 @@ void Collection::search_candidates(uint32_t* filter_ids, size_t filter_ids_lengt
}
}
-void Collection::union_with_filter_ids(std::vector<const art_leaf*> & leaves, uint32_t** filter_ids,
- uint32_t & filter_ids_length) {
+size_t Collection::union_of_leaf_ids(std::vector<const art_leaf *> &leaves, uint32_t **results_out) {
+ uint32_t *results = nullptr;
+ size_t results_length = 0;
- for(const art_leaf* leaf: leaves) {
- uint32_t* results = new uint32_t[filter_ids_length + leaf->values->ids.getLength()];
- size_t results_length = leaf->values->ids.do_union(*filter_ids, filter_ids_length, results);
+ uint32_t *prev_results = nullptr;
+ size_t prev_results_length = 0;
- delete [] *filter_ids;
+ for(const art_leaf* leaf: leaves) {
+ results_length = leaf->values->ids.do_union(prev_results, prev_results_length, &results);
- *filter_ids = results;
- filter_ids_length = results_length;
+ delete [] prev_results;
+ prev_results = results;
+ prev_results_length = results_length;
}
+
+ *results_out = results;
+ return results_length;
}
nlohmann::json Collection::search(std::string query, const std::vector<std::string> fields, const std::vector<filter> filters,
@@ -275,25 +280,31 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
art_tree* t = index_map.at(a_filter.field_name);
field f = schema.at(a_filter.field_name);
- nlohmann::json json_value = nlohmann::json::parse(a_filter.value_json);
if(f.type == field_types::INT64) {
- } else if(f.type == field_types::INT32) {
- int32_t value = json_value.get<int32_t>();
- NUM_COMPARATOR comparator = a_filter.get_comparator();
-
+ } else if(f.type == field_types::INT32 || f.type == field_types::INT32_ARRAY) {
std::vector<const art_leaf*> leaves;
- art_int32_search(t, value, comparator, leaves);
- union_with_filter_ids(leaves, &filter_ids, filter_ids_length);
- } else if(f.type == field_types::INT32_ARRAY) {
- int32_t value = json_value.get<int32_t>();
- NUM_COMPARATOR comparator = a_filter.get_comparator();
+ for(const std::string & filter_value: a_filter.values) {
+ int32_t value = (int32_t) std::stoi(filter_value);
+ NUM_COMPARATOR comparator = a_filter.get_comparator();
+ art_int32_search(t, value, comparator, leaves);
+ }
- std::vector<const art_leaf*> leaves;
- art_int32_search(t, value, comparator, leaves);
- union_with_filter_ids(leaves, &filter_ids, filter_ids_length);
+ uint32_t* result_ids = nullptr;
+ size_t result_ids_length = union_of_leaf_ids(leaves, &result_ids);
+
+ if(filter_ids == nullptr) {
+ filter_ids = result_ids;
+ filter_ids_length = result_ids_length;
+ } else {
+ uint32_t* filtered_results = new uint32_t[std::min((size_t)filter_ids_length, result_ids_length)];
+ filter_ids_length = Intersection::scalar(filter_ids, filter_ids_length, result_ids, result_ids_length, filtered_results);
+ delete [] filter_ids;
+ delete [] result_ids;
+ filter_ids = filtered_results;
+ }
}
}
}
@@ -314,6 +325,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
}
}
+ delete [] filter_ids;
+
std::sort(field_order_kvs.begin(), field_order_kvs.end(),
[](const std::pair<int, Topster<100>::KV> & a, const std::pair<int, Topster<100>::KV> & b) {
if(a.second.match_score != b.second.match_score) return a.second.match_score > b.second.match_score;
diff --git a/src/sorted_array.cpp b/src/sorted_array.cpp
index b9384dca..b02ccd1c 100644
--- a/src/sorted_array.cpp
+++ b/src/sorted_array.cpp
@@ -169,20 +169,26 @@ void sorted_array::remove_values(uint32_t *sorted_values, uint32_t values_length
delete[] new_array;
}
-size_t sorted_array::intersect(uint32_t* arr, const size_t arr_length, uint32_t* results) {
+size_t sorted_array::intersect(uint32_t* arr, const size_t arr_length, uint32_t** results_out) {
uint32_t* curr = uncompress();
+ uint32_t* results = new uint32_t[std::min(arr_length, (size_t) length)];
+
size_t results_length = Intersection::scalar(arr, arr_length, curr, length, results);
delete[] curr;
+
+ *results_out = results;
return results_length;
}
-size_t sorted_array::do_union(uint32_t *arr, const size_t arr_length, uint32_t *results) {
+size_t sorted_array::do_union(uint32_t *arr, const size_t arr_length, uint32_t** results_out) {
size_t curr_index = 0, arr_index = 0, res_index = 0;
uint32_t* curr = uncompress();
+ uint32_t* results = new uint32_t[arr_length+length];
if(arr == nullptr) {
memcpy(results, curr, length * sizeof(uint32_t));
delete[] curr;
+ *results_out = results;
return length;
}
@@ -221,5 +227,7 @@ size_t sorted_array::do_union(uint32_t *arr, const size_t arr_length, uint32_t *
}
delete[] curr;
+
+ *results_out = results;
return res_index;
}
\ No newline at end of file
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 0ecd8246..4f275676 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -378,7 +378,7 @@ TEST_F(CollectionTest, MultipleFields) {
}
}
-TEST_F(CollectionTest, SearchNumericFields) {
+TEST_F(CollectionTest, SearchInt32Fields) {
Collection *coll_array_fields;
std::ifstream infile(std::string(ROOT_DIR)+"test/numeric_array_documents.jsonl");
@@ -415,7 +415,7 @@ TEST_F(CollectionTest, SearchNumericFields) {
}
// Searching on an int32 field
- std::vector<filter> filters = {(filter) {"age", "24", "GREATER_THAN"}};
+ std::vector<filter> filters = {(filter) {"age", {"24"}, "GREATER_THAN"}};
results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
@@ -429,18 +429,16 @@ TEST_F(CollectionTest, SearchNumericFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- filters = {(filter) {"age", "24", "GREATER_THAN_EQUALS"}};
+ filters = {(filter) {"age", {"24"}, "GREATER_THAN_EQUALS"}};
results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
ASSERT_EQ(4, results["hits"].size());
- filters = {(filter) {"age", "24", "EQUALS"}};
+ filters = {(filter) {"age", {"24"}, "EQUALS"}};
results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
ASSERT_EQ(1, results["hits"].size());
// Searching a number against an int32 array field
- //std::cout << "int32 array field:" << std::endl;
-
- filters = {(filter) {"years", "2002", "GREATER_THAN"}};
+ filters = {(filter) {"years", {"2002"}, "GREATER_THAN"}};
results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
@@ -452,7 +450,7 @@ TEST_F(CollectionTest, SearchNumericFields) {
ASSERT_STREQ(id.c_str(), result_id.c_str());
}
- filters = {(filter) {"years", "1989", "LESS_THAN"}};
+ filters = {(filter) {"years", {"1989"}, "LESS_THAN"}};
results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
ASSERT_EQ(1, results["hits"].size());
@@ -462,18 +460,44 @@ TEST_F(CollectionTest, SearchNumericFields) {
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
- //std::cout << result_id << std::endl;
}
- /*search_fields = {"cast"};
- results = coll_array_fields->search("chris pine", search_fields, {}, 0, 10, FREQUENCY, false);
+ // multiple filters
+ filters = {(filter) {"years", {"2005"}, "LESS_THAN"}, (filter) {"years", {"1987"}, "GREATER_THAN"}};
+ results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(1, results["hits"].size());
+
+ ids = {"4"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // multiple search values (works like SQL's IN operator) against a single int field
+ filters = {(filter) {"age", {"21", "24", "63"}, "EQUALS"}};
+ results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
ASSERT_EQ(3, results["hits"].size());
- ids = {"7", "6", "1"};
+ ids = {"3", "0", "2"};
+ for(size_t i = 0; i < results["hits"].size(); i++) {
+ nlohmann::json result = results["hits"].at(i);
+ std::string result_id = result["id"];
+ std::string id = ids.at(i);
+ ASSERT_STREQ(id.c_str(), result_id.c_str());
+ }
+
+ // multiple search values against an int array field
+ filters = {(filter) {"years", {"2015", "1985", "1999"}, "EQUALS"}};
+ results = coll_array_fields->search("Jeremy", search_fields, filters, 0, 10, FREQUENCY, false);
+ ASSERT_EQ(4, results["hits"].size());
+
+ ids = {"3", "1", "4", "0"};
for(size_t i = 0; i < results["hits"].size(); i++) {
nlohmann::json result = results["hits"].at(i);
std::string result_id = result["id"];
std::string id = ids.at(i);
ASSERT_STREQ(id.c_str(), result_id.c_str());
- }*/
+ }
}
diff --git a/test/sorted_array_test.cpp b/test/sorted_array_test.cpp
index ea5c8d62..b3388eb6 100644
--- a/test/sorted_array_test.cpp
+++ b/test/sorted_array_test.cpp
@@ -109,8 +109,8 @@ TEST(SortedArrayTest, Intersect) {
}
// arr1: [0..8] , arr2: [3, 6, 9]
- uint32_t *results = new uint32_t[size2];
- uint32_t results_size = arr1.intersect(arr2, arr2_len, results);
+ uint32_t *results = nullptr;
+ uint32_t results_size = arr1.intersect(arr2, arr2_len, &results);
ASSERT_EQ(2, results_size);
std::vector<uint32_t> expected = {3, 6};
@@ -118,6 +118,8 @@ TEST(SortedArrayTest, Intersect) {
for(size_t i = 0; i < results_size; i++) {
ASSERT_EQ(expected[i], results[i]);
}
+
+ delete [] results;
}
TEST(SortedArrayTest, MergeShouldRemoveDuplicates) {
@@ -137,8 +139,8 @@ TEST(SortedArrayTest, MergeShouldRemoveDuplicates) {
}
// arr1: [0..8] , arr2: [3, 6, 9]
- uint32_t *results = new uint32_t[size1];
- uint32_t results_size = arr1.do_union(arr2, arr2_len, results);
+ uint32_t *results = nullptr;
+ uint32_t results_size = arr1.do_union(arr2, arr2_len, &results);
ASSERT_EQ(10, results_size);
std::vector<uint32_t> expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
@@ -164,8 +166,8 @@ TEST(SortedArrayTest, MergeShouldRemoveDuplicatesAtBoundary) {
arr2[j++] = i;
}
- uint32_t *results = new uint32_t[10];
- uint32_t results_size = arr1.do_union(arr2, vec2.size(), results);
+ uint32_t *results = nullptr;
+ uint32_t results_size = arr1.do_union(arr2, vec2.size(), &results);
ASSERT_EQ(9, results_size);
std::vector<uint32_t> expected = {0, 1, 2, 3, 4, 5, 6, 7, 8};