Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Support locale and symbols_to_index in synonyms.

Kishore Nallan940c2b982d89
diff --git a/include/synonym_index.h b/include/synonym_index.h
index 4b978e90..995be21e 100644
--- a/include/synonym_index.h
+++ b/include/synonym_index.h
@@ -12,6 +12,8 @@ struct synonym_t {
std::string id;
std::vector<std::string> root;
std::vector<std::vector<std::string>> synonyms;
+ std::string locale;
+ std::vector<char> symbols;
synonym_t() = default;
@@ -21,62 +23,9 @@ struct synonym_t {
}
- nlohmann::json to_view_json() const {
- nlohmann::json obj;
- obj["id"] = id;
- obj["root"] = StringUtils::join(root, " ");
+ nlohmann::json to_view_json() const;
- obj["synonyms"] = nlohmann::json::array();
-
- for(const auto& synonym: synonyms) {
- obj["synonyms"].push_back(StringUtils::join(synonym, " "));
- }
-
- return obj;
- }
-
- static Option<bool> parse(const nlohmann::json& synonym_json, synonym_t& syn) {
- if(synonym_json.count("id") == 0) {
- return Option<bool>(400, "Missing `id` field.");
- }
-
- if(synonym_json.count("synonyms") == 0) {
- return Option<bool>(400, "Could not find an array of `synonyms`");
- }
-
- if(synonym_json.count("root") != 0 && !synonym_json["root"].is_string()) {
- return Option<bool>(400, "Key `root` should be a string.");
- }
-
- if (!synonym_json["synonyms"].is_array() || synonym_json["synonyms"].empty()) {
- return Option<bool>(400, "Could not find an array of `synonyms`");
- }
-
- for(const auto& synonym: synonym_json["synonyms"]) {
- if(!synonym.is_string() || synonym == "") {
- return Option<bool>(400, "Could not find a valid string array of `synonyms`");
- }
-
- std::vector<std::string> tokens;
- StringUtils::split(synonym, tokens, " ");
- for(auto& token: tokens) {
- StringUtils::tolowercase(token);
- }
- syn.synonyms.push_back(tokens);
- }
-
- if(synonym_json.count("root") != 0) {
- std::vector<std::string> tokens;
- StringUtils::split(synonym_json["root"], tokens, " ");
- for(auto& token: tokens) {
- StringUtils::tolowercase(token);
- }
- syn.root = tokens;
- }
-
- syn.id = synonym_json["id"];
- return Option<bool>(true);
- }
+ static Option<bool> parse(const nlohmann::json& synonym_json, synonym_t& syn);
static uint64_t get_hash(const std::vector<std::string>& tokens) {
uint64_t hash = 1;
diff --git a/src/synonym_index.cpp b/src/synonym_index.cpp
index 24b96f4e..eeb40e43 100644
--- a/src/synonym_index.cpp
+++ b/src/synonym_index.cpp
@@ -184,3 +184,87 @@ spp::sparse_hash_map<std::string, synonym_t> SynonymIndex::get_synonyms() {
std::string SynonymIndex::get_synonym_key(const std::string & collection_name, const std::string & synonym_id) {
return std::string(COLLECTION_SYNONYM_PREFIX) + "_" + collection_name + "_" + synonym_id;
}
+
+Option<bool> synonym_t::parse(const nlohmann::json& synonym_json, synonym_t& syn) {
+ if(synonym_json.count("id") == 0) {
+ return Option<bool>(400, "Missing `id` field.");
+ }
+
+ if(synonym_json.count("synonyms") == 0) {
+ return Option<bool>(400, "Could not find an array of `synonyms`");
+ }
+
+ if(synonym_json.count("root") != 0 && !synonym_json["root"].is_string()) {
+ return Option<bool>(400, "Key `root` should be a string.");
+ }
+
+ if (!synonym_json["synonyms"].is_array() || synonym_json["synonyms"].empty()) {
+ return Option<bool>(400, "Could not find an array of `synonyms`");
+ }
+
+ if(synonym_json.count("locale") != 0) {
+ if(!synonym_json["locale"].is_string()) {
+ return Option<bool>(400, "Synonym `locale` should be a string.`");
+ }
+
+ syn.locale = synonym_json["locale"].get<std::string>();
+ }
+
+ if(synonym_json.count("symbols_to_index") != 0) {
+ if(!synonym_json["symbols_to_index"].is_array() || synonym_json["symbols_to_index"].empty() ||
+ !synonym_json["symbols_to_index"][0].is_string()) {
+ return Option<bool>(400, "Synonym `symbols_to_index` should be an array of strings.");
+ }
+
+ auto symbols = synonym_json["symbols_to_index"].get<std::vector<std::string>>();
+ for(auto symbol: symbols) {
+ if(symbol.size() != 1) {
+ return Option<bool>(400, "Synonym `symbols_to_index` should be an array of single character symbols.");
+ }
+
+ syn.symbols.push_back(symbol[0]);
+ }
+ }
+
+ if(synonym_json.count("root") != 0) {
+ std::vector<std::string> tokens;
+ Tokenizer(synonym_json["root"], true, false, syn.locale, syn.symbols).tokenize(tokens);
+ syn.root = tokens;
+ }
+
+ for(const auto& synonym: synonym_json["synonyms"]) {
+ if(!synonym.is_string() || synonym == "") {
+ return Option<bool>(400, "Could not find a valid string array of `synonyms`");
+ }
+
+ std::vector<std::string> tokens;
+ Tokenizer(synonym, true, false, syn.locale, syn.symbols).tokenize(tokens);
+ syn.synonyms.push_back(tokens);
+ }
+
+ syn.id = synonym_json["id"];
+ return Option<bool>(true);
+}
+
+nlohmann::json synonym_t::to_view_json() const {
+ nlohmann::json obj;
+ obj["id"] = id;
+ obj["root"] = StringUtils::join(root, " ");
+
+ obj["synonyms"] = nlohmann::json::array();
+
+ for(const auto& synonym: synonyms) {
+ obj["synonyms"].push_back(StringUtils::join(synonym, " "));
+ }
+
+ if(!locale.empty()) {
+ obj["locale"] = locale;
+ }
+
+ if(!symbols.empty()) {
+ obj["symbols"] = symbols;
+ }
+
+ return obj;
+}
+
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index 50086f2d..cf49f4f9 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -83,7 +83,8 @@ TEST_F(CollectionSynonymsTest, SynonymParsingFromJson) {
nlohmann::json syn_plus_json = {
{"id", "syn-plus"},
{"root", "+"},
- {"synonyms", {"plus", "#"} }
+ {"synonyms", {"plus", "#"} },
+ {"symbols_to_index", {"+", "#"}},
};
synonym_t synonym_plus;
@@ -169,6 +170,29 @@ TEST_F(CollectionSynonymsTest, SynonymParsingFromJson) {
syn_op = synonym_t::parse(syn_json_root_bad_type, synonym);
ASSERT_FALSE(syn_op.ok());
ASSERT_STREQ("Key `root` should be a string.", syn_op.error().c_str());
+
+ // bad symbols to index
+ nlohmann::json syn_json_bad_symbols = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", {"Sea"} },
+ {"symbols_to_index", {}}
+ };
+
+ syn_op = synonym_t::parse(syn_json_bad_symbols, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Synonym `symbols_to_index` should be an array of strings.", syn_op.error().c_str());
+
+ syn_json_bad_symbols = {
+ {"id", "syn-1"},
+ {"root", "Ocean"},
+ {"synonyms", {"Sea"} },
+ {"symbols_to_index", {"%^"}}
+ };
+
+ syn_op = synonym_t::parse(syn_json_bad_symbols, synonym);
+ ASSERT_FALSE(syn_op.ok());
+ ASSERT_STREQ("Synonym `symbols_to_index` should be an array of single character symbols.", syn_op.error().c_str());
}
TEST_F(CollectionSynonymsTest, SynonymReductionOneWay) {
@@ -510,8 +534,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {
ASSERT_EQ(2, res["hits"].size());
ASSERT_EQ(2, res["found"].get<uint32_t>());
- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
// for now we don't support synonyms on ANY prefix
@@ -889,7 +913,8 @@ TEST_F(CollectionSynonymsTest, HandleSpecialSymbols) {
nlohmann::json syn_plus_json = {
{"id", "syn-1"},
{"root", "plus"},
- {"synonyms", {"+"} }
+ {"synonyms", {"+"} },
+ {"symbols_to_index", {"+"}}
};
ASSERT_TRUE(coll1->add_synonym(syn_plus_json).ok());
@@ -936,4 +961,53 @@ TEST_F(CollectionSynonymsTest, SynonymForNonAsciiLanguage) {
auto res = coll1->search("எல்லோருக்கும்", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, res["hits"].size());
ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionSynonymsTest, SynonymForKorean) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string", "locale": "ko"},
+ {"name": "points", "type": "int32" }
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"도쿄구울", "100"},
+ {"도쿄 구울", "100"},
+ {"구울", "100"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["points"] = std::stoi(records[i][1]);
+
+ auto add_op = coll1->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ nlohmann::json synonym1 = R"({
+ "id": "syn-1",
+ "root": "",
+ "synonyms": ["도쿄구울", "도쿄 구울", "구울"],
+ "locale": "ko"
+ })"_json;
+
+ ASSERT_TRUE(coll1->add_synonym(synonym1).ok());
+
+ auto res = coll1->search("도쿄구울", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(3, res["hits"].size());
+
+ res = coll1->search("도쿄 구울", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(3, res["hits"].size());
+
+ res = coll1->search("구울", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(3, res["hits"].size());
+}