Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Buffer IDs during filtering.
Kishore Nallan44edeb8a5e8c
diff --git a/include/collection.h b/include/collection.hindex 52ebb431..f0bd2ff2 100644--- a/include/collection.h+++ b/include/collection.h@@ -191,7 +191,8 @@ private:static std::vector<char> to_char_array(const std::vector<std::string>& strs);Option<bool> validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,- std::vector<sort_by>& sort_fields_std) const;+ std::vector<sort_by>& sort_fields_std,+ bool is_wildcard_query) const;Option<bool> persist_collection_meta();diff --git a/src/collection.cpp b/src/collection.cppindex b5aa5afc..8ee7527d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -523,7 +523,8 @@ void Collection::curate_results(string& actual_query, const string& filter_query}Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<sort_by> & sort_fields,- std::vector<sort_by>& sort_fields_std) const {+ std::vector<sort_by>& sort_fields_std,+ const bool is_wildcard_query) const {size_t num_sort_expressions = 0;@@ -718,7 +719,10 @@ Option<bool> Collection::validate_and_standardize_sort_fields(const std::vector<4. THREE: do nothing*/if(sort_fields_std.empty()) {- sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);+ if(!is_wildcard_query) {+ sort_fields_std.emplace_back(sort_field_const::text_match, sort_field_const::desc);+ }+if(!default_sorting_field.empty()) {sort_fields_std.emplace_back(default_sorting_field, sort_field_const::desc);} else {@@ -1140,8 +1144,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,sort_fields_guard_t sort_fields_guard;std::vector<sort_by>& sort_fields_std = sort_fields_guard.sort_fields_std;+ bool is_wildcard_query = (query == "*");+if(curated_sort_by.empty()) {- auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields, sort_fields_std);+ auto sort_validation_op = validate_and_standardize_sort_fields(sort_fields, sort_fields_std, is_wildcard_query);if(!sort_validation_op.ok()) {return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());}@@ -1152,7 +1158,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,return Option<nlohmann::json>(400, "Parameter `sort_by` is malformed.");}- auto sort_validation_op = validate_and_standardize_sort_fields(curated_sort_fields, sort_fields_std);+ auto sort_validation_op = validate_and_standardize_sort_fields(curated_sort_fields, sort_fields_std,+ is_wildcard_query);if(!sort_validation_op.ok()) {return Option<nlohmann::json>(sort_validation_op.code(), sort_validation_op.error());}diff --git a/src/index.cpp b/src/index.cppindex ccf0e532..2481064d 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1808,18 +1808,17 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,} else if(f.is_string()) {art_tree* t = search_index.at(a_filter.field_name);- uint32_t* ids = nullptr;- size_t ids_size = 0;+ uint32_t* or_ids = nullptr;+ size_t or_ids_size = 0;- for(const std::string & filter_value: a_filter.values) {- uint32_t* strt_ids = nullptr;- size_t strt_ids_size = 0;+ // aggregates IDs across array of filter values and reduces excessive ORing+ std::vector<uint32_t> f_id_buff;+ for(const std::string & filter_value: a_filter.values) {std::vector<void*> posting_lists;// there could be multiple tokens in a filter value, which we have to treat as ANDs// e.g. country: South Africa-Tokenizer tokenizer(filter_value, true, false, f.locale, symbols_to_index, token_separators);std::string str_token;@@ -1838,72 +1837,97 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,posting_lists.push_back(leaf->values);}- // For NOT_EQUALS alone, it is okay for none of the results to match prior to negation- // e.g. field:!= [RANDOM_NON_EXISTING_STRING]- if(a_filter.comparators[0] != NOT_EQUALS && posting_lists.size() != str_tokens.size()) {+ if(posting_lists.size() != str_tokens.size()) {continue;}- std::vector<uint32_t> result_id_vec;- posting_t::intersect(posting_lists, result_id_vec);- if(!result_id_vec.empty()) {- strt_ids = new uint32_t [result_id_vec.size()];- std::copy(result_id_vec.begin(), result_id_vec.end(), strt_ids);- strt_ids_size = result_id_vec.size();- }-if(a_filter.comparators[0] == EQUALS || a_filter.comparators[0] == NOT_EQUALS) {- // need to do exact match (unlike CONTAINS)- uint32_t* exact_strt_ids = new uint32_t[strt_ids_size];- size_t exact_strt_size = 0;+ // needs intersection + exact matching (unlike CONTAINS)+ std::vector<uint32_t> result_id_vec;+ posting_t::intersect(posting_lists, result_id_vec);- posting_t::get_exact_matches(posting_lists, f.is_array(), strt_ids, strt_ids_size,- exact_strt_ids, exact_strt_size);+ if(result_id_vec.empty()) {+ continue;+ }- delete[] strt_ids;- strt_ids = exact_strt_ids;- strt_ids_size = exact_strt_size;- }+ // need to do exact match+ uint32_t* exact_str_ids = new uint32_t[result_id_vec.size()];+ size_t exact_str_ids_size = 0;+ std::unique_ptr<uint32_t[]> exact_str_ids_guard(exact_str_ids);- if(a_filter.comparators[0] == NOT_EQUALS) {- // exclude records from existing IDs (from previous filters or ALL records)- // upstream will guarantee that NOT_EQUALS is placed right at the end of filters list- uint32_t* excluded_strt_ids = nullptr;- size_t excluded_strt_size = 0;+ posting_t::get_exact_matches(posting_lists, f.is_array(), result_id_vec.data(), result_id_vec.size(),+ exact_str_ids, exact_str_ids_size);- if(ids == nullptr) {- if(filter_ids == nullptr) {- ids = seq_ids->uncompress();- ids_size = seq_ids->num_ids();- } else {- ids = filter_ids;- ids_size = filter_ids_length;- }+ if(exact_str_ids_size == 0) {+ continue;}- excluded_strt_size = ArrayUtils::exclude_scalar(ids, ids_size, strt_ids,- strt_ids_size, &excluded_strt_ids);-- if(filter_ids == nullptr) {- // means we had to uncompress `seq_ids` so need to free that- delete [] ids;+ for(size_t ei = 0; ei < exact_str_ids_size; ei++) {+ f_id_buff.push_back(exact_str_ids[ei]);}- ids = excluded_strt_ids;- ids_size = excluded_strt_size;- delete[] strt_ids;} else {- // Otherwise, we just ensure that given record contains tokens in the filter query+ // CONTAINS+ size_t before_size = f_id_buff.size();+ posting_t::intersect(posting_lists, f_id_buff);+ if(f_id_buff.size() == before_size) {+ continue;+ }+ }++ if(f_id_buff.size() > 100000 || a_filter.values.size() == 1) {+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());+uint32_t* out = nullptr;- ids_size = ArrayUtils::or_scalar(ids, ids_size, strt_ids, strt_ids_size, &out);- delete[] strt_ids;- delete[] ids;- ids = out;+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);+ delete[] or_ids;+ or_ids = out;+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory+ }+ }++ if(!f_id_buff.empty()) {+ gfx::timsort(f_id_buff.begin(), f_id_buff.end());+ f_id_buff.erase(std::unique( f_id_buff.begin(), f_id_buff.end() ), f_id_buff.end());++ uint32_t* out = nullptr;+ or_ids_size = ArrayUtils::or_scalar(or_ids, or_ids_size, f_id_buff.data(), f_id_buff.size(), &out);+ delete[] or_ids;+ or_ids = out;+ std::vector<uint32_t>().swap(f_id_buff); // clears out memory+ }++ if(a_filter.comparators[0] == NOT_EQUALS) {+ // exclude records from existing IDs (from previous filters or ALL records)+ // "not equals" can only be applied to the entire array so we can do the exclusion operations once here+ uint32_t* excluded_strt_ids = nullptr;+ size_t excluded_strt_size = 0;++ if(result_ids == nullptr) {+ if(filter_ids == nullptr) {+ result_ids = seq_ids->uncompress();+ result_ids_len = seq_ids->num_ids();+ } else {+ result_ids = filter_ids;+ result_ids_len = filter_ids_length;+ }}++ excluded_strt_size = ArrayUtils::exclude_scalar(result_ids, result_ids_len, or_ids,+ or_ids_size, &excluded_strt_ids);++ if(filter_ids == nullptr) {+ // means we had to uncompress `seq_ids` so need to free that+ delete [] result_ids;+ }++ or_ids = excluded_strt_ids;+ or_ids_size = excluded_strt_size;}- result_ids = ids;- result_ids_len = ids_size;+ result_ids = or_ids;+ result_ids_len = or_ids_size;}if(i == 0) {@@ -2459,6 +2483,12 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vconst uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0);const std::string& field = the_fields[0].name;+ // if filters were not provided, use the seq_ids index to generate the list of all document ids+ if(filters.empty() && filter_ids_length == 0) {+ filter_ids_length = seq_ids->num_ids();+ filter_ids = seq_ids->uncompress();+ }+curate_filtered_ids(filters, curated_ids, excluded_result_ids,excluded_result_ids_size, filter_ids, filter_ids_length, curated_ids_sorted);collate_included_ids({}, included_ids_map, curated_topster, searched_queries);@@ -4040,13 +4070,6 @@ void Index::curate_filtered_ids(const std::vector<filter>& filters, const std::sconst uint32_t* exclude_token_ids, size_t exclude_token_ids_size,uint32_t*& filter_ids, uint32_t& filter_ids_length,const std::vector<uint32_t>& curated_ids_sorted) const {- // if filtered results are not available, use the seq_ids index to generate the list of all document ids-- if(filters.empty() && filter_ids_length == 0) {- filter_ids_length = seq_ids->num_ids();- filter_ids = seq_ids->uncompress();- }-if(!curated_ids.empty()) {uint32_t *excluded_result_ids = nullptr;filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0],diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 464c4f5f..e8587544 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -1843,6 +1843,10 @@ TEST_F(CollectionFilteringTest, NegationOperatorBasics) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_STREQ("3", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ // when no such value exists: should return all results+ results = coll1->search("*", {"artist"}, "artist:!=Foobar", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 10).get();+ ASSERT_EQ(4, results["found"].get<size_t>());+// empty value (bad filtering)auto res_op = coll1->search("*", {"artist"}, "artist:!=", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 10);ASSERT_FALSE(res_op.ok());diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 278c39ef..207da563 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -1269,3 +1269,25 @@ TEST_F(CollectionSpecificMoreTest, HighlightObjectShouldBeEmptyWhenNoHighlightFiASSERT_TRUE(res["hits"][0]["highlight"]["snippet"].empty());}++TEST_F(CollectionSpecificMoreTest, WildcardSearchWithNoSortingField) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "Sample Title";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto res_op = coll1->search("*", {}, "", {}, {}, {2}, 10, 1,+ FREQUENCY, {true});++ ASSERT_TRUE(res_op.ok());+ auto res = res_op.get();+ ASSERT_EQ(1, res["hits"].size());+}