Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Filtering on string field should be verbatim by default.
kishorencaa9e4a226e57
diff --git a/include/art.h b/include/art.hindex 1d3a286d..9c6276d0 100644--- a/include/art.h+++ b/include/art.h@@ -130,6 +130,7 @@ enum NUM_COMPARATOR {LESS_THAN,LESS_THAN_EQUALS,EQUALS,+ CONTAINS,GREATER_THAN,GREATER_THAN_EQUALS};diff --git a/include/index.h b/include/index.hindex ce2995f0..b146a2c0 100644--- a/include/index.h+++ b/include/index.h@@ -236,7 +236,7 @@ public:void score_results(const std::vector<sort_by> & sort_fields, const uint16_t & query_index, const uint8_t & field_id,const uint32_t total_cost, Topster* topster, const std::vector<art_leaf *> & query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size) const;+ const uint32_t *result_ids, const size_t result_size);static int32_t get_points_from_doc(const nlohmann::json &document, const std::string & default_sorting_field);@@ -288,5 +288,11 @@ public:static int64_t float_to_in64_t(float n);uint64_t get_distinct_id(const std::unordered_map<std::string, size_t> &facet_to_id, const uint32_t seq_id) const;++ void get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index);++ void eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,+ const std::vector<art_leaf *> &query_suggestion,+ uint32_t *exact_strt_ids, size_t& exact_strt_size) const;};diff --git a/src/collection.cpp b/src/collection.cppindex 4ebd6de3..72469b49 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -610,12 +610,21 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}} else if(_field.is_string()) {- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ size_t filter_value_index = 0;+ NUM_COMPARATOR str_comparator = EQUALS;++ if(raw_value[0] == '~') {+ // string filter should be evaluated in "contains" mode+ str_comparator = CONTAINS;+ while(raw_value[++filter_value_index] == ' ');+ }++ if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {std::vector<std::string> filter_values;StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");- f = {field_name, filter_values, EQUALS};+ f = {field_name, filter_values, str_comparator};} else {- f = {field_name, {raw_value}, EQUALS};+ f = {field_name, {raw_value}, str_comparator};}} else {return Option<nlohmann::json>(400, "Error with field `" + _field.name + "`: Unidentified field type.");diff --git a/src/index.cpp b/src/index.cppindex 41f82f7e..a55775f9 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -96,6 +96,7 @@ Option<uint32_t> Index::index_in_memory(const nlohmann::json &document, uint32_t}// initialize facet index since it will be updated as well during search indexing+ // even if a field is optional, a facet position will be available in the vector for that fieldstd::vector<std::vector<uint64_t>> values(facet_schema.size());facet_index_v2.emplace(seq_id, values);@@ -630,12 +631,7 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const uint32_t* result_ids, size_t results_size) {std::unordered_map<std::string, size_t> facet_to_index;-- size_t i_facet = 0;- for(const auto & facet: facet_schema) {- facet_to_index[facet.first] = i_facet;- i_facet++;- }+ get_facet_to_index(facet_to_index);// assumed that facet fields have already been validated upstreamfor(auto & a_facet: facets) {@@ -781,6 +777,14 @@ void Index::do_facets(std::vector<facet> & facets, facet_query_t & facet_query,}}+void Index::get_facet_to_index(std::unordered_map<std::string, size_t>& facet_to_index) {+ size_t i_facet = 0;+ for(const auto & facet_kv: facet_schema) {+ facet_to_index.emplace(facet_kv.first, i_facet);+ i_facet++;+ }+}+void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, size_t filter_ids_length,const std::vector<uint32_t>& curated_ids,const std::vector<sort_by> & sort_fields,@@ -884,6 +888,9 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectouint32_t* filter_ids = nullptr;uint32_t filter_ids_length = 0;+ std::unordered_map<std::string, size_t> facet_to_index;+ get_facet_to_index(facet_to_index);+for(size_t i = 0; i < filters.size(); i++) {const filter & a_filter = filters[i];if(search_index.count(a_filter.field_name) == 0) {@@ -945,9 +952,11 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectouint32_t* strt_ids = nullptr;size_t strt_ids_size = 0;+ std::vector<art_leaf *> query_suggestion;+// there could be multiple tokens in a filter value, which we have to treat as ANDs// e.g. country: South Africa- for(auto & str_token : str_tokens) {+ for(auto& str_token: str_tokens) {string_utils.unicode_normalize(str_token);art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),@@ -956,6 +965,8 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectocontinue;}+ query_suggestion.push_back(leaf);+if(strt_ids == nullptr) {strt_ids = leaf->values->ids.uncompress();strt_ids_size = leaf->values->ids.getLength();@@ -971,6 +982,24 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vecto}}+ if(a_filter.compare_operator == EQUALS) {+ // need to do exact match (unlike CONTAINS) by iterating the result ids and calculating scores+ if(query_suggestion.size() == str_tokens.size()) {+ uint32_t *exact_strt_ids = new uint32_t[strt_ids_size];+ size_t exact_strt_size = 0;++ eq_str_filter_plain(strt_ids, strt_ids_size, query_suggestion, exact_strt_ids, exact_strt_size);++ delete[] strt_ids;+ strt_ids = exact_strt_ids;+ strt_ids_size = exact_strt_size;+ }+ }++ // (NOT implemented) if the query is wrapped by double quotes, ensure exact match+ // bool exact_match = (filter_value.front() == '"' && filter_value.back() == '"');++ // Otherwise, we just ensure that given record contains tokens in the filter queryuint32_t* out = nullptr;ids_size = ArrayUtils::or_scalar(ids, ids_size, strt_ids, strt_ids_size, &out);delete[] strt_ids;@@ -999,6 +1028,59 @@ Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vectoreturn Option<>(filter_ids_length);}+void Index::eq_str_filter_plain(const uint32_t *strt_ids, size_t strt_ids_size,+ const std::vector<art_leaf *> &query_suggestion, uint32_t *exact_strt_ids,+ size_t& exact_strt_size) const {+ spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;+ for (art_leaf *token_leaf: query_suggestion) {+ if(token_leaf == nullptr) {+ leaf_to_indices.emplace(token_leaf, nullptr);+ continue;+ }++ uint32_t *indices = new uint32_t[strt_ids_size];+ token_leaf->values->ids.indexOf(strt_ids, strt_ids_size, indices);+ leaf_to_indices.emplace(token_leaf, indices);+ }++ // e.g. First In First Out => hash([0, 1, 0, 2])+ spp::sparse_hash_map<size_t, uint32_t> leaf_to_id;+ size_t next_id = 1;+ size_t filter_hash = 1;++ for(size_t leaf_index=0; leaf_index<query_suggestion.size(); leaf_index++) {+ if(leaf_to_id.count(leaf_index) == 0) {+ leaf_to_id.emplace(leaf_index, next_id++);+ }++ uint32_t leaf_id = leaf_to_id[leaf_index];+ filter_hash *= (1779033703 + 2*leaf_id*(leaf_index+1));+ }++ for(size_t strt_ids_index = 0; strt_ids_index < strt_ids_size; strt_ids_index++) {+ std::vector<std::vector<std::vector<uint16_t>>> array_token_positions;+ populate_token_positions(query_suggestion, leaf_to_indices, strt_ids_index, array_token_positions);+ // iterate array_token_positions and compute hash++ for(size_t array_index = 0; array_index < array_token_positions.size(); array_index++) {+ auto& token_positions = array_token_positions[array_index];+ size_t this_hash = 1;++ for(size_t token_index = 0; token_index < token_positions.size(); token_index++) {+ auto& positions = token_positions[token_index];+ for(auto pos: positions) {+ this_hash *= (1779033703 + 2*(token_index+1)*(pos+1));+ }+ }++ if(this_hash == filter_hash) {+ exact_strt_ids[exact_strt_size++] = strt_ids[strt_ids_index];+ break;+ }+ }+ }+}+uint32_t* Index::collate_leaf_ids(const std::vector<const art_leaf *> &leaves, size_t& result_ids_len) const {std::vector<uint32_t> ids;@@ -1421,16 +1503,21 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16const uint8_t & field_id, const uint32_t total_cost, Topster* topster,const std::vector<art_leaf *> &query_suggestion,spp::sparse_hash_set<uint64_t>& groups_processed,- const uint32_t *result_ids, const size_t result_size) const {+ const uint32_t *result_ids, const size_t result_size) {spp::sparse_hash_map<const art_leaf*, uint32_t*> leaf_to_indices;-for (art_leaf *token_leaf : query_suggestion) {uint32_t *indices = new uint32_t[result_size];token_leaf->values->ids.indexOf(result_ids, result_size, indices);leaf_to_indices.emplace(token_leaf, indices);}+ std::unordered_map<std::string, size_t> facet_to_index;+ get_facet_to_index(facet_to_index);++ Match single_token_match = Match(1, 0);+ const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);+int sort_order[3]; // 1 or -1 based on DESC or ASC respectivelyspp::sparse_hash_map<uint32_t, int64_t>* field_values[3];@@ -1447,19 +1534,6 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16//auto begin = std::chrono::high_resolution_clock::now();- Match single_token_match = Match(1, 0);- const uint64_t single_token_match_score = single_token_match.get_match_score(total_cost, field_id);-- std::unordered_map<std::string, size_t> facet_to_id;-- if(search_params->group_limit > 0) {- size_t i_facet = 0;- for(const auto & facet: facet_schema) {- facet_to_id[facet.first] = i_facet;- i_facet++;- }- }-for(size_t i=0; i<result_size; i++) {const uint32_t seq_id = result_ids[i];@@ -1537,7 +1611,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16uint64_t distinct_id = seq_id;if(search_params->group_limit != 0) {- distinct_id = get_distinct_id(facet_to_id, seq_id);+ distinct_id = get_distinct_id(facet_to_index, seq_id);groups_processed.emplace(distinct_id);}diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 041f7f07..6c1e8835 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -7,10 +7,11 @@std::string lower_and_no_special_chars(const std::string & str) {std::stringstream ss;- for(const auto c: str) {- bool should_remove = ( (int)(c) >= 0 && // check for ASCII range- !std::isalnum(c) );- if(!should_remove) {+ for(char c : str) {+ bool is_ascii = ((int)(c) >= 0);+ bool keep_char = !is_ascii || std::isalnum(c);++ if(keep_char) {ss << (char) std::tolower(c);}}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 82db30a4..daf51598 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -140,6 +140,23 @@ TEST_F(CollectionFacetingTest, FacetCounts) {ASSERT_STREQ("FINE PLATINUM", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());ASSERT_STREQ("bronze", results["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());+ // facet with filter on string array field must fail when partial token is used+ facets.clear();+ facets.push_back("tags");+ results = coll_array_fields->search("Jeremy", query_fields, "tags: PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(0, results["hits"].size());+ ASSERT_EQ(0, results["found"].get<size_t>());++ // partial token should be used along with ~+ results = coll_array_fields->search("Jeremy", query_fields, "tags:~ PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());++ // or when tokens match facet value exactly+ results = coll_array_fields->search("Jeremy", query_fields, "tags: FINE PLATINUM", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["found"].get<size_t>());+// facet with wildcard queryfacets.clear();facets.push_back("tags");diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 3eee6ddf..14033a57 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -847,7 +847,7 @@ TEST_F(CollectionTest, MultipleFields) {// filtering on unfaceted multi-valued string fieldquery_fields = {"title"};- results = coll_mul_fields->search("captain", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();+ results = coll_mul_fields->search("captain", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1, FREQUENCY, false).get();ASSERT_EQ(1, results["hits"].size());ids = {"6"};for(size_t i = 0; i < results["hits"].size(); i++) {@@ -928,7 +928,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {// filtering on facet field should be possible (supports partial word search but without typo tolerance)query_fields = {"title"};- result_op = coll_mul_fields->search("*", query_fields, "cast: chris", facets, sort_fields, 0, 10, 1,+ result_op = coll_mul_fields->search("*", query_fields, "cast:~ chris", facets, sort_fields, 0, 10, 1,FREQUENCY, false);ASSERT_EQ(true, result_op.ok());results = result_op.get();