Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Buffer IDs during filtering.

Kishore Nallan44edeb8a5e8c
diff --git a/include/collection.h b/include/collection.h
index 52ebb431..f0bd2ff2 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -191,7 +191,8 @@ private:
static std::vector<char> to_char_array(const std::vector<std::string>& strs);
Option<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,
- std::vector<sort_by>& sort_fields_std) const;
+ std::vector<sort_by>& sort_fields_std,
+ bool is_wildcard_query) const;
Option<bool> persist_collection_meta();
diff --git a/src/collection.cpp b/src/collection.cpp
index b5aa5afc..8ee7527d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -523,7 +523,8 @@ void Collection::curate_results(string& actual_query, const string& filter_query
}
Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,
- std::vector<sort_by>& sort_fields_std) const {
+ std::vector<sort_by>& sort_fields_std,
+ const bool is_wildcard_query) const {
size_t num_sort_expressions = 0;
@@ -718,7 +719,10 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<
4. THREE: do nothing
*/
if(sort_fields_std.empty()) {
- sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);
+ if(!is_wildcard_query) {
+ sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);
+ }
+
if(!default_sorting_field.empty()) {
sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);
} else {
@@ -1140,8 +1144,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
sort_fields_guard_t sort_fields_guard;
std::vector<sort_by>& sort_fields_std = sort_fields_guard.sort_fields_std;
+ bool is_wildcard_query = (query == "*");
+
if(curated_sort_by.empty()) {
- auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields, sort_fields_std);
+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields, sort_fields_std, is_wildcard_query);
if(!sort_validation_op.ok()) {
return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());
}
@@ -1152,7 +1158,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
return Option<nlohmann::json>(400, "Parameter `sort_by` is malformed.");
}
- auto sort_validation_op = validate_and_standardize_sort_fields(curated_sort_fields, sort_fields_std);
+ auto sort_validation_op = validate_and_standardize_sort_fields(curated_sort_fields, sort_fields_std,
+ is_wildcard_query);
if(!sort_validation_op.ok()) {
return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());
}
diff --git a/src/index.cpp b/src/index.cpp
index ccf0e532..2481064d 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1808,18 +1808,17 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
} else if(f.is_string()) {
art_tree* t = search_index.at(a_filter.field_name);
- uint32_t* ids = nullptr;
- size_t ids_size = 0;
+ uint32_t* or_ids = nullptr;
+ size_t or_ids_size = 0;
- for(const std::string & filter_value: a_filter.values) {
- uint32_t* strt_ids = nullptr;
- size_t strt_ids_size = 0;
+ // aggregates IDs across array of filter values and reduces excessive ORing
+ std::vector<uint32_t> f_id_buff;
+ for(const std::string & filter_value: a_filter.values) {
std::vector<void*> posting_lists;
// there could be multiple tokens in a filter value, which we have to treat as ANDs
// e.g. country: South Africa
-
Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);
std::string str_token;
@@ -1838,72 +1837,97 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,
posting_lists.push_back(leaf->values);
}
- // For NOT_EQUALS alone, it is okay for none of the results to match prior to negation
- // e.g. field:!= [RANDOM_NON_EXISTING_STRING]
- if(a_filter.comparators[0] != NOT_EQUALS && posting_lists.size() != str_tokens.size()) {
+ if(posting_lists.size() != str_tokens.size()) {
continue;
}
- std::vector<uint32_t> result_id_vec;
- posting_t::intersect(posting_lists, result_id_vec);
- if(!result_id_vec.empty()) {
- strt_ids = new uint32_t [result_id_vec.size()];
- std::copy(result_id_vec.begin(), result_id_vec.end(), strt_ids);
- strt_ids_size = result_id_vec.size();
- }
-
if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {
- // need to do exact match (unlike CONTAINS)
- uint32_t* exact_strt_ids = new uint32_t[strt_ids_size];
- size_t exact_strt_size = 0;
+ // needs intersection + exact matching (unlike CONTAINS)
+ std::vector<uint32_t> result_id_vec;
+ posting_t::intersect(posting_lists, result_id_vec);
- posting_t::get_exact_matches(posting_lists, f.is_array(), strt_ids, strt_ids_size,
- exact_strt_ids, exact_strt_size);
+ if(result_id_vec.empty()) {
+ continue;
+ }
- delete[] strt_ids;
- strt_ids = exact_strt_ids;
- strt_ids_size = exact_strt_size;
- }
+ // need to do exact match
+ uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];
+ size_t exact_str_ids_size = 0;
+ std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);
- if(a_filter.comparators[0] == NOT_EQUALS) {
- // exclude records from existing IDs (from previous filters or ALL records)
- // upstream will guarantee that NOT_EQUALS is placed right at the end of filters list
- uint32_t* excluded_strt_ids = nullptr;
- size_t excluded_strt_size = 0;
+ posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),
+ exact_str_ids, exact_str_ids_size);
- if(ids == nullptr) {
- if(filter_ids == nullptr) {
- ids = seq_ids->uncompress();
- ids_size = seq_ids->num_ids();
- } else {
- ids = filter_ids;
- ids_size = filter_ids_length;
- }
+ if(exact_str_ids_size == 0) {
+ continue;
}
- excluded_strt_size = ArrayUtils::exclude_scalar(ids, ids_size, strt_ids,
- strt_ids_size, &excluded_strt_ids);
-
- if(filter_ids == nullptr) {
- // means we had to uncompress `seq_ids` so need to free that
- delete [] ids;
+ for(size_t ei = 0; ei < exact_str_ids_size; ei++) {
+ f_id_buff.push_back(exact_str_ids[ei]);
}
- ids = excluded_strt_ids;
- ids_size = excluded_strt_size;
- delete[] strt_ids;
} else {
- // Otherwise, we just ensure that given record contains tokens in the filter query
+ // CONTAINS
+ size_t before_size = f_id_buff.size();
+ posting_t::intersect(posting_lists, f_id_buff);
+ if(f_id_buff.size() == before_size) {
+ continue;
+ }
+ }
+
+ if(f_id_buff.size() > 100000 || a_filter.values.size() == 1) {
+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());
+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
+
uint32_t* out = nullptr;
- ids_size = ArrayUtils::or_scalar(ids, ids_size, strt_ids, strt_ids_size, &out);
- delete[] strt_ids;
- delete[] ids;
- ids = out;
+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
+ delete[] or_ids;
+ or_ids = out;
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ }
+ }
+
+ if(!f_id_buff.empty()) {
+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());
+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());
+
+ uint32_t* out = nullptr;
+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);
+ delete[] or_ids;
+ or_ids = out;
+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory
+ }
+
+ if(a_filter.comparators[0] == NOT_EQUALS) {
+ // exclude records from existing IDs (from previous filters or ALL records)
+ // "not equals" can only be applied to the entire array so we can do the exclusion operations once here
+ uint32_t* excluded_strt_ids = nullptr;
+ size_t excluded_strt_size = 0;
+
+ if(result_ids == nullptr) {
+ if(filter_ids == nullptr) {
+ result_ids = seq_ids->uncompress();
+ result_ids_len = seq_ids->num_ids();
+ } else {
+ result_ids = filter_ids;
+ result_ids_len = filter_ids_length;
+ }
}
+
+ excluded_strt_size = ArrayUtils::exclude_scalar(result_ids, result_ids_len, or_ids,
+ or_ids_size, &excluded_strt_ids);
+
+ if(filter_ids == nullptr) {
+ // means we had to uncompress `seq_ids` so need to free that
+ delete [] result_ids;
+ }
+
+ or_ids = excluded_strt_ids;
+ or_ids_size = excluded_strt_size;
}
- result_ids = ids;
- result_ids_len = ids_size;
+ result_ids = or_ids;
+ result_ids_len = or_ids_size;
}
if(i == 0) {
@@ -2459,6 +2483,12 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);
const std::string& field = the_fields[0].name;
+ // if filters were not provided, use the seq_ids index to generate the list of all document ids
+ if(filters.empty() && filter_ids_length == 0) {
+ filter_ids_length = seq_ids->num_ids();
+ filter_ids = seq_ids->uncompress();
+ }
+
curate_filtered_ids(filters, curated_ids, excluded_result_ids,
excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);
collate_included_ids({}, included_ids_map, curated_topster, searched_queries);
@@ -4040,13 +4070,6 @@ void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::s
const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
uint32_t*& filter_ids, uint32_t& filter_ids_length,
const std::vector<uint32_t>& curated_ids_sorted) const {
- // if filtered results are not available, use the seq_ids index to generate the list of all document ids
-
- if(filters.empty() && filter_ids_length == 0) {
- filter_ids_length = seq_ids->num_ids();
- filter_ids = seq_ids->uncompress();
- }
-
if(!curated_ids.empty()) {
uint32_t *excluded_result_ids = nullptr;
filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 464c4f5f..e8587544 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -1843,6 +1843,10 @@ TEST_F(CollectionFilteringTest, NegationOperatorBasics) {
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_STREQ("3", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ // when no such value exists: should return all results
+ results = coll1->search("*", {"artist"}, "artist:!=Foobar", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 10).get();
+ ASSERT_EQ(4, results["found"].get<size_t>());
+
// empty value (bad filtering)
auto res_op = coll1->search("*", {"artist"}, "artist:!=", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 10);
ASSERT_FALSE(res_op.ok());
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 278c39ef..207da563 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1269,3 +1269,25 @@ TEST_F(CollectionSpecificMoreTest, HighlightObjectShouldBeEmptyWhenNoHighlightFi
ASSERT_TRUE(res["hits"][0]["highlight"]["snippet"].empty());
}
+
+TEST_F(CollectionSpecificMoreTest, WildcardSearchWithNoSortingField) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Sample Title";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto res_op = coll1->search("*", {}, "", {}, {}, {2}, 10, 1,
+ FREQUENCY, {true});
+
+ ASSERT_TRUE(res_op.ok());
+ auto res = res_op.get();
+ ASSERT_EQ(1, res["hits"].size());
+}