Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Process filters before synonyms are resolved.
Kishore Nallan769e59402354
diff --git a/include/index.h b/include/index.hindex 6d4ab7d7..d0f012a7 100644--- a/include/index.h+++ b/include/index.h@@ -493,11 +493,6 @@ private:bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,std::vector<filter>& filters) const;- void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,- std::vector<query_tokens_t>& field_query_tokens,- token_ordering token_order,- std::vector<filter>& filters) const;-bool resolve_override(const std::vector<std::string>& rule_tokens, bool exact_rule_match,const std::vector<std::string>& query_tokens,token_ordering token_order, std::set<std::string>& absorbed_tokens,@@ -857,6 +852,11 @@ public:const std::vector<search_field_t>& the_fields, Topster* topster,const size_t num_search_fields,spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const;++ void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,+ std::vector<std::string>& query_tokens,+ token_ordering token_order,+ std::vector<filter>& filters) const;};template<class T>diff --git a/src/collection.cpp b/src/collection.cppindex 60e95521..51e08cdc 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -984,27 +984,30 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sfield_query_tokens[0].q_exclude_tokens, field_query_tokens[0].q_phrases, "",false);} else {- for(size_t i = 0; i < search_fields.size(); i++) {- const auto& search_field = search_fields[i];- field_query_tokens.emplace_back(query_tokens_t{});+ field_query_tokens.emplace_back(query_tokens_t{});+ const std::string & field_locale = search_schema.at(search_fields[0]).locale;+ parse_search_query(query, field_query_tokens[0].q_include_tokens,+ field_query_tokens[0].q_exclude_tokens,+ field_query_tokens[0].q_phrases,+ field_locale, pre_segmented_query);- const std::string & field_locale = search_schema.at(search_field).locale;- parse_search_query(query, field_query_tokens[i].q_include_tokens,- field_query_tokens[i].q_exclude_tokens,- field_query_tokens[i].q_phrases,- field_locale, pre_segmented_query);-- if(i == 0) {- q_tokens = field_query_tokens[i].q_include_tokens;- for(auto& phrase: field_query_tokens[i].q_phrases) {- for(auto& token: phrase) {- q_tokens.push_back(token);- }- }+ // process filter overrides first, before synonyms (order is important)+ index->process_filter_overrides(filter_overrides, field_query_tokens[0].q_include_tokens, token_order, filters);++ // get synonyms+ synonym_reduction(field_query_tokens[0].q_include_tokens, field_query_tokens[0].q_synonyms);++ q_tokens = field_query_tokens[0].q_include_tokens;++ for(auto& phrase: field_query_tokens[0].q_phrases) {+ for(auto& token: phrase) {+ q_tokens.push_back(token);}+ }- // get synonyms- synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);+ for(size_t i = 1; i < search_fields.size(); i++) {+ field_query_tokens.emplace_back(query_tokens_t{});+ field_query_tokens[i] = field_query_tokens[0];}}diff --git a/src/index.cpp b/src/index.cppindex e023bdf7..1a441d39 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1842,50 +1842,28 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const}void Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,- std::vector<query_tokens_t>& field_query_tokens,+ std::vector<std::string>& query_tokens,token_ordering token_order,std::vector<filter>& filters) const {+ std::shared_lock lock(mutex);size_t orig_filters_size = filters.size();for(auto& override: filter_overrides) {if(!override->rule.dynamic_query) {// Simple static filtering: add to filter_by and rewrite query if needed.// Check the original query and then the synonym variants until a rule matches.- bool resolved_override = static_filter_query_eval(override, field_query_tokens[0].q_include_tokens, filters);-- if(!resolved_override) {- // we have not been able to resolve an override, so look at synonyms- for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {- static_filter_query_eval(override, syn_tokens, filters);- if(orig_filters_size != filters.size()) {- // we have been able to resolve an override via synonym, so can stop looking- resolved_override = true;- break;- }- }- }+ bool resolved_override = static_filter_query_eval(override, query_tokens, filters);- if(resolved_override && override->remove_matched_tokens) {- std::vector<std::string>& tokens = field_query_tokens[0].q_include_tokens;+ if(resolved_override) {+ if(!override->remove_matched_tokens) {+ return ;+ }std::vector<std::string> rule_tokens;Tokenizer(override->rule.query, true).tokenize(rule_tokens);std::set<std::string> rule_token_set(rule_tokens.begin(), rule_tokens.end());-- remove_matched_tokens(tokens, rule_token_set);-- for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {- remove_matched_tokens(syn_tokens, rule_token_set);- }-- // copy over for other fields- for(size_t i = 1; i < field_query_tokens.size(); i++) {- field_query_tokens[i] = field_query_tokens[0];- }- }-- if(resolved_override) {+ remove_matched_tokens(query_tokens, rule_token_set);return ;}} else {@@ -1895,50 +1873,23 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filtestd::vector<std::string> rule_parts;StringUtils::split(override->rule.query, rule_parts, " ");- std::vector<std::string>& query_tokens = field_query_tokens[0].q_include_tokens;-uint32_t* field_override_ids = nullptr;size_t field_override_ids_len = 0;bool exact_rule_match = override->rule.match == override_t::MATCH_EXACT;-std::string filter_by_clause = override->filter_by;std::set<std::string> absorbed_tokens;- bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens, token_order,- absorbed_tokens, filter_by_clause);-- if(!resolved_override) {- // try resolving synonym-- for(size_t i = 0; i < field_query_tokens[0].q_synonyms.size(); i++) {- absorbed_tokens.clear();- std::vector<std::string>& syn_tokens = field_query_tokens[0].q_synonyms[i];- resolved_override = resolve_override(rule_parts, exact_rule_match, syn_tokens, token_order,- absorbed_tokens, filter_by_clause);-- if(resolved_override) {- break;- }- }- }+ bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens,+ token_order, absorbed_tokens, filter_by_clause);if(resolved_override) {Option<bool> filter_parse_op = filter::parse_filter_query(filter_by_clause, search_schema, store, "",filters);if(filter_parse_op.ok()) {if(override->remove_matched_tokens) {- std::vector<std::string>& tokens = field_query_tokens[0].q_include_tokens;+ std::vector<std::string>& tokens = query_tokens;remove_matched_tokens(tokens, absorbed_tokens);-- for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {- remove_matched_tokens(syn_tokens, absorbed_tokens);- }-- // copy over for other fields- for(size_t i = 1; i < field_query_tokens.size(); i++) {- field_query_tokens[i] = field_query_tokens[0];- }}}@@ -2115,7 +2066,6 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::vstd::shared_lock lock(mutex);- process_filter_overrides(filter_overrides, field_query_tokens, token_order, filters);do_filtering(filter_ids, filter_ids_length, filters, true);std::vector<uint32_t> included_ids_vec;diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex acd7097e..e5e54e76 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -1035,8 +1035,8 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());- ASSERT_EQ("<mark>shoes</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());- ASSERT_EQ("<mark>shoes</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["highlights"].size());+ ASSERT_EQ(0, results["hits"][1]["highlights"].size());// should not apply filter for non-exact caseresults = coll1->search("running shoes", {"name", "category", "brand"}, "",@@ -1686,16 +1686,6 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {ASSERT_EQ(2, results["found"].get<uint32_t>());ASSERT_EQ(2, results["hits"].size());- // with synonum for expensive- synonym_t synonym1{"costly-expensive", {"costly"}, {{"expensive"}} };- coll1->add_synonym(synonym1);-- results = coll1->search("costly", {"name"}, "",- {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();-- ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());-// with exact matchresults = coll1->search("cheap", {"name"}, "",@@ -1711,6 +1701,82 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {ASSERT_EQ(0, results["hits"].size());+ // with synonym for expensive: should NOT match as synonyms are resolved after override substitution+ synonym_t synonym1{"costly-expensive", {"costly"}, {{"expensive"}} };+ coll1->add_synonym(synonym1);++ results = coll1->search("costly", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(0, results["hits"].size());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionOverrideTest, SynonymsAppliedToOverridenQuery) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING, false),+ field("price", field_types::FLOAT, true),+ field("points", field_types::INT32, false)};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "Amazing Shoes";+ doc1["price"] = 399.99;+ doc1["points"] = 3;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "White Sneakers";+ doc2["price"] = 149.99;+ doc2["points"] = 5;++ nlohmann::json doc3;+ doc3["id"] = "2";+ doc3["name"] = "Red Sneakers";+ doc3["price"] = 49.99;+ doc3["points"] = 5;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());++ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };++ nlohmann::json override_json_contains = {+ {"id", "static-filters"},+ {+ "rule", {+ {"query", "expensive"},+ {"match", override_t::MATCH_CONTAINS}+ }+ },+ {"remove_matched_tokens", true},+ {"filter_by", "price:> 100"}+ };++ override_t override_contains;+ auto op = override_t::parse(override_json_contains, "static-filters", override_contains);+ ASSERT_TRUE(op.ok());++ coll1->add_override(override_contains);++ synonym_t synonym1{"shoes-sneakers", {"shoes"}, {{"sneakers"}} };+ coll1->add_synonym(synonym1);++ auto results = coll1->search("expensive shoes", {"name"}, "",+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+collectionManager.drop_collection("coll1");}