Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Process filters before synonyms are resolved.

Kishore Nallan769e59402354
diff --git a/include/index.h b/include/index.h
index 6d4ab7d7..d0f012a7 100644
--- a/include/index.h
+++ b/include/index.h
@@ -493,11 +493,6 @@ private:
bool static_filter_query_eval(const override_t* override, std::vector<std::string>& tokens,
std::vector<filter>& filters) const;
- void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
- std::vector<query_tokens_t>& field_query_tokens,
- token_ordering token_order,
- std::vector<filter>& filters) const;
-
bool resolve_override(const std::vector<std::string>& rule_tokens, bool exact_rule_match,
const std::vector<std::string>& query_tokens,
token_ordering token_order, std::set<std::string>& absorbed_tokens,
@@ -857,6 +852,11 @@ public:
const std::vector<search_field_t>& the_fields, Topster* topster,
const size_t num_search_fields,
spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const;
+
+ void process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
+ std::vector<std::string>& query_tokens,
+ token_ordering token_order,
+ std::vector<filter>& filters) const;
};
template<class T>
diff --git a/src/collection.cpp b/src/collection.cpp
index 60e95521..51e08cdc 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -984,27 +984,30 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
field_query_tokens[0].q_exclude_tokens, field_query_tokens[0].q_phrases, "",
false);
} else {
- for(size_t i = 0; i < search_fields.size(); i++) {
- const auto& search_field = search_fields[i];
- field_query_tokens.emplace_back(query_tokens_t{});
+ field_query_tokens.emplace_back(query_tokens_t{});
+ const std::string & field_locale = search_schema.at(search_fields[0]).locale;
+ parse_search_query(query, field_query_tokens[0].q_include_tokens,
+ field_query_tokens[0].q_exclude_tokens,
+ field_query_tokens[0].q_phrases,
+ field_locale, pre_segmented_query);
- const std::string & field_locale = search_schema.at(search_field).locale;
- parse_search_query(query, field_query_tokens[i].q_include_tokens,
- field_query_tokens[i].q_exclude_tokens,
- field_query_tokens[i].q_phrases,
- field_locale, pre_segmented_query);
-
- if(i == 0) {
- q_tokens = field_query_tokens[i].q_include_tokens;
- for(auto& phrase: field_query_tokens[i].q_phrases) {
- for(auto& token: phrase) {
- q_tokens.push_back(token);
- }
- }
+ // process filter overrides first, before synonyms (order is important)
+ index->process_filter_overrides(filter_overrides, field_query_tokens[0].q_include_tokens, token_order, filters);
+
+ // get synonyms
+ synonym_reduction(field_query_tokens[0].q_include_tokens, field_query_tokens[0].q_synonyms);
+
+ q_tokens = field_query_tokens[0].q_include_tokens;
+
+ for(auto& phrase: field_query_tokens[0].q_phrases) {
+ for(auto& token: phrase) {
+ q_tokens.push_back(token);
}
+ }
- // get synonyms
- synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);
+ for(size_t i = 1; i < search_fields.size(); i++) {
+ field_query_tokens.emplace_back(query_tokens_t{});
+ field_query_tokens[i] = field_query_tokens[0];
}
}
diff --git a/src/index.cpp b/src/index.cpp
index e023bdf7..1a441d39 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1842,50 +1842,28 @@ bool Index::resolve_override(const std::vector<std::string>& rule_tokens, const
}
void Index::process_filter_overrides(const std::vector<const override_t*>& filter_overrides,
- std::vector<query_tokens_t>& field_query_tokens,
+ std::vector<std::string>& query_tokens,
token_ordering token_order,
std::vector<filter>& filters) const {
+ std::shared_lock lock(mutex);
size_t orig_filters_size = filters.size();
for(auto& override: filter_overrides) {
if(!override->rule.dynamic_query) {
// Simple static filtering: add to filter_by and rewrite query if needed.
// Check the original query and then the synonym variants until a rule matches.
- bool resolved_override = static_filter_query_eval(override, field_query_tokens[0].q_include_tokens, filters);
-
- if(!resolved_override) {
- // we have not been able to resolve an override, so look at synonyms
- for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {
- static_filter_query_eval(override, syn_tokens, filters);
- if(orig_filters_size != filters.size()) {
- // we have been able to resolve an override via synonym, so can stop looking
- resolved_override = true;
- break;
- }
- }
- }
+ bool resolved_override = static_filter_query_eval(override, query_tokens, filters);
- if(resolved_override && override->remove_matched_tokens) {
- std::vector<std::string>& tokens = field_query_tokens[0].q_include_tokens;
+ if(resolved_override) {
+ if(!override->remove_matched_tokens) {
+ return ;
+ }
std::vector<std::string> rule_tokens;
Tokenizer(override->rule.query, true).tokenize(rule_tokens);
std::set<std::string> rule_token_set(rule_tokens.begin(), rule_tokens.end());
-
- remove_matched_tokens(tokens, rule_token_set);
-
- for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {
- remove_matched_tokens(syn_tokens, rule_token_set);
- }
-
- // copy over for other fields
- for(size_t i = 1; i < field_query_tokens.size(); i++) {
- field_query_tokens[i] = field_query_tokens[0];
- }
- }
-
- if(resolved_override) {
+ remove_matched_tokens(query_tokens, rule_token_set);
return ;
}
} else {
@@ -1895,50 +1873,23 @@ void Index::process_filter_overrides(const std::vector<const override_t*>& filte
std::vector<std::string> rule_parts;
StringUtils::split(override->rule.query, rule_parts, " ");
- std::vector<std::string>& query_tokens = field_query_tokens[0].q_include_tokens;
-
uint32_t* field_override_ids = nullptr;
size_t field_override_ids_len = 0;
bool exact_rule_match = override->rule.match == override_t::MATCH_EXACT;
-
std::string filter_by_clause = override->filter_by;
std::set<std::string> absorbed_tokens;
- bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens, token_order,
- absorbed_tokens, filter_by_clause);
-
- if(!resolved_override) {
- // try resolving synonym
-
- for(size_t i = 0; i < field_query_tokens[0].q_synonyms.size(); i++) {
- absorbed_tokens.clear();
- std::vector<std::string>& syn_tokens = field_query_tokens[0].q_synonyms[i];
- resolved_override = resolve_override(rule_parts, exact_rule_match, syn_tokens, token_order,
- absorbed_tokens, filter_by_clause);
-
- if(resolved_override) {
- break;
- }
- }
- }
+ bool resolved_override = resolve_override(rule_parts, exact_rule_match, query_tokens,
+ token_order, absorbed_tokens, filter_by_clause);
if(resolved_override) {
Option<bool> filter_parse_op = filter::parse_filter_query(filter_by_clause, search_schema, store, "",
filters);
if(filter_parse_op.ok()) {
if(override->remove_matched_tokens) {
- std::vector<std::string>& tokens = field_query_tokens[0].q_include_tokens;
+ std::vector<std::string>& tokens = query_tokens;
remove_matched_tokens(tokens, absorbed_tokens);
-
- for(auto& syn_tokens: field_query_tokens[0].q_synonyms) {
- remove_matched_tokens(syn_tokens, absorbed_tokens);
- }
-
- // copy over for other fields
- for(size_t i = 1; i < field_query_tokens.size(); i++) {
- field_query_tokens[i] = field_query_tokens[0];
- }
}
}
@@ -2115,7 +2066,6 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
std::shared_lock lock(mutex);
- process_filter_overrides(filter_overrides, field_query_tokens, token_order, filters);
do_filtering(filter_ids, filter_ids_length, filters, true);
std::vector<uint32_t> included_ids_vec;
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index acd7097e..e5e54e76 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -1035,8 +1035,8 @@ TEST_F(CollectionOverrideTest, DynamicFilteringExactMatchBasics) {
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
- ASSERT_EQ("<mark>shoes</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
- ASSERT_EQ("<mark>shoes</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["highlights"].size());
+ ASSERT_EQ(0, results["hits"][1]["highlights"].size());
// should not apply filter for non-exact case
results = coll1->search("running shoes", {"name", "category", "brand"}, "",
@@ -1686,16 +1686,6 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {
ASSERT_EQ(2, results["found"].get<uint32_t>());
ASSERT_EQ(2, results["hits"].size());
- // with synonum for expensive
- synonym_t synonym1{"costly-expensive", {"costly"}, {{"expensive"}} };
- coll1->add_synonym(synonym1);
-
- results = coll1->search("costly", {"name"}, "",
- {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
-
- ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
-
// with exact match
results = coll1->search("cheap", {"name"}, "",
@@ -1711,6 +1701,82 @@ TEST_F(CollectionOverrideTest, StaticFiltering) {
ASSERT_EQ(0, results["hits"].size());
+ // with synonym for expensive: should NOT match as synonyms are resolved after override substitution
+ synonym_t synonym1{"costly-expensive", {"costly"}, {{"expensive"}} };
+ coll1->add_synonym(synonym1);
+
+ results = coll1->search("costly", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(0, results["hits"].size());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionOverrideTest, SynonymsAppliedToOverridenQuery) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("price", field_types::FLOAT, true),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Amazing Shoes";
+ doc1["price"] = 399.99;
+ doc1["points"] = 3;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "White Sneakers";
+ doc2["price"] = 149.99;
+ doc2["points"] = 5;
+
+ nlohmann::json doc3;
+ doc3["id"] = "2";
+ doc3["name"] = "Red Sneakers";
+ doc3["price"] = 49.99;
+ doc3["points"] = 5;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc3.dump()).ok());
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ nlohmann::json override_json_contains = {
+ {"id", "static-filters"},
+ {
+ "rule", {
+ {"query", "expensive"},
+ {"match", override_t::MATCH_CONTAINS}
+ }
+ },
+ {"remove_matched_tokens", true},
+ {"filter_by", "price:> 100"}
+ };
+
+ override_t override_contains;
+ auto op = override_t::parse(override_json_contains, "static-filters", override_contains);
+ ASSERT_TRUE(op.ok());
+
+ coll1->add_override(override_contains);
+
+ synonym_t synonym1{"shoes-sneakers", {"shoes"}, {{"sneakers"}} };
+ coll1->add_synonym(synonym1);
+
+ auto results = coll1->search("expensive shoes", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
collectionManager.drop_collection("coll1");
}