Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Tokenize/normalize override query properly.

Kishore Nallanc7a7df676b8e
diff --git a/include/override.h b/include/override.h
index a92c01c2..2900f866 100644
--- a/include/override.h
+++ b/include/override.h
@@ -45,7 +45,12 @@ struct override_t {
override_t() = default;
- static Option<bool> parse(const nlohmann::json& override_json, const std::string& id, override_t& override);
+ static Option<bool> parse(const nlohmann::json& override_json, const std::string& id,
+ override_t& override,
+ const std::string& locale = "",
+ const std::vector<char>& symbols_to_index = {},
+ const std::vector<char>& token_separators = {}
+ );
nlohmann::json to_json() const;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 362088ef..458df719 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -643,10 +643,18 @@ void Collection::curate_results(string& actual_query, const string& filter_query
}
}
- std::string query = actual_query;
if(enable_overrides && !overrides.empty()) {
- StringUtils::tolowercase(query);
+ std::string query;
+
+ if(actual_query == "*") {
+ query = "*";
+ } else {
+ std::vector<std::string> tokens;
+ Tokenizer tokenizer(actual_query, true, false, "", symbols_to_index, token_separators);
+ tokenizer.tokenize(tokens);
+ query = StringUtils::join(tokens, " ");
+ }
for(const auto& override_kv: overrides) {
const auto& override = override_kv.second;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 2155f2f5..1c7e912b 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1311,7 +1311,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
for(const auto & collection_override_json: collection_override_jsons) {
nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);
override_t override;
- auto parse_op = override_t::parse(collection_override, "", override);
+ auto parse_op = override_t::parse(collection_override, "", override, "", collection->get_symbols_to_index(),
+ collection->get_token_separators());
if(parse_op.ok()) {
collection->add_override(override, false);
} else {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index b67b6465..e3b17d6a 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1399,7 +1399,9 @@ bool put_override(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht
}
override_t override;
- Option<bool> parse_op = override_t::parse(req_json, override_id, override);
+ Option<bool> parse_op = override_t::parse(req_json, override_id, override, "",
+ collection->get_symbols_to_index(),
+ collection->get_token_separators());
if(!parse_op.ok()) {
res->set(parse_op.code(), parse_op.error());
return false;
diff --git a/src/override.cpp b/src/override.cpp
index 96f96d38..572189d5 100644
--- a/src/override.cpp
+++ b/src/override.cpp
@@ -2,7 +2,11 @@
#include "override.h"
#include "tokenizer.h"
-Option<bool> override_t::parse(const nlohmann::json& override_json, const std::string& id, override_t& override) {
+Option<bool> override_t::parse(const nlohmann::json& override_json, const std::string& id,
+ override_t& override,
+ const std::string& locale,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators) {
if(!override_json.is_object()) {
return Option<bool>(400, "Bad JSON.");
}
@@ -110,8 +114,14 @@ Option<bool> override_t::parse(const nlohmann::json& override_json, const std::s
override.rule.match = json_rule.count("match") == 0 ? "" : json_rule["match"].get<std::string>();
if(!override.rule.query.empty()) {
- override.rule.normalized_query = override.rule.query;
- Tokenizer::normalize_ascii(override.rule.normalized_query);
+ auto symbols = symbols_to_index;
+ symbols.push_back('{');
+ symbols.push_back('}');
+ symbols.push_back('*');
+ Tokenizer tokenizer(override.rule.query, true, false, locale, symbols, token_separators);
+ std::vector<std::string> tokens;
+ tokenizer.tokenize(tokens);
+ override.rule.normalized_query = StringUtils::join(tokens, " ");
}
if(json_rule.count("filter_by") != 0) {
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index b602c777..f5b368ec 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -948,6 +948,50 @@ TEST_F(CollectionOverrideTest, RuleQueryMustBeCaseInsensitive) {
ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
}
+TEST_F(CollectionOverrideTest, RuleQueryWithAccentedChars) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("color", field_types::STRING, false),
+ field("points", field_types::INT32, false)};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Green";
+ doc1["color"] = "Green";
+ doc1["points"] = 30;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ std::vector<sort_by> sort_fields = { sort_by("_text_match", "DESC"), sort_by("points", "DESC") };
+
+ nlohmann::json override_json = R"({
+ "id": "rule-1",
+ "rule": {
+ "query": "Grün",
+ "match": "contains"
+ },
+ "filter_by":"color:green",
+ "filter_curated_hits":true
+ })"_json;
+
+ override_t override_rule;
+ auto op = override_t::parse(override_json, "rule-1", override_rule);
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override_rule);
+
+ auto results = coll1->search("grün", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+}
+
TEST_F(CollectionOverrideTest, WindowForRule) {
Collection *coll1;
@@ -1428,7 +1472,7 @@ TEST_F(CollectionOverrideTest, PinnedHitsWhenThereAreNotEnoughResults) {
// multiple pinned hits specified, but query produces no result
- auto results = coll_mul_fields->search("notfoundquery", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
+ auto results = coll_mul_fields->search("not-foundquery", {"title"}, "", {"starring"}, {}, {0}, 10, 1, FREQUENCY,
{false}, Index::DROP_TOKENS_THRESHOLD,
spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "starring: will", 30, 5,
@@ -3158,3 +3202,80 @@ TEST_F(CollectionOverrideTest, DynamicFilteringWithPartialTokenMatch) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionOverrideTest, OverrideWithSymbolsToIndex) {
+ Collection* coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING, false),
+ field("category", field_types::STRING, true),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "", static_cast<uint64_t>(std::time(nullptr)),
+ "", {"-"}, {}).get();
+ }
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "Non-Stick";
+ doc1["category"] = "Cookware";
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "NonStick";
+ doc2["category"] = "Kitchen";
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {sort_by("_text_match", "DESC")};
+
+ auto results = coll1->search("non-stick", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+
+ // with override, we return all records
+
+ nlohmann::json override_json = {
+ {"id", "ov-1"},
+ {
+ "rule", {
+ {"query", "non-stick"},
+ {"match", override_t::MATCH_EXACT}
+ }
+ },
+ {"filter_by", "category:= Cookware"}
+ };
+
+ override_t override;
+ auto op = override_t::parse(override_json, "ov-1", override, "", {'-'}, {});
+ ASSERT_TRUE(op.ok());
+ coll1->add_override(override);
+
+ results = coll1->search("non-stick", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ results = coll1->search("nonstick", {"name"}, "",
+ {}, sort_fields, {2}, 10, 1, FREQUENCY,
+ {false}, Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "", 10).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+
+ collectionManager.drop_collection("coll1");
+}
+