Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support locale and symbols_to_index in synonyms.
Kishore Nallan940c2b982d89
diff --git a/include/synonym_index.h b/include/synonym_index.hindex 4b978e90..995be21e 100644--- a/include/synonym_index.h+++ b/include/synonym_index.h@@ -12,6 +12,8 @@ struct synonym_t {std::string id;std::vector<std::string> root;std::vector<std::vector<std::string>> synonyms;+ std::string locale;+ std::vector<char> symbols;synonym_t() = default;@@ -21,62 +23,9 @@ struct synonym_t {}- nlohmann::json to_view_json() const {- nlohmann::json obj;- obj["id"] = id;- obj["root"] = StringUtils::join(root, " ");+ nlohmann::json to_view_json() const;- obj["synonyms"] = nlohmann::json::array();-- for(const auto& synonym: synonyms) {- obj["synonyms"].push_back(StringUtils::join(synonym, " "));- }-- return obj;- }-- static Option<bool> parse(const nlohmann::json& synonym_json, synonym_t& syn) {- if(synonym_json.count("id") == 0) {- return Option<bool>(400, "Missing `id` field.");- }-- if(synonym_json.count("synonyms") == 0) {- return Option<bool>(400, "Could not find an array of `synonyms`");- }-- if(synonym_json.count("root") != 0 && !synonym_json["root"].is_string()) {- return Option<bool>(400, "Key `root` should be a string.");- }-- if (!synonym_json["synonyms"].is_array() || synonym_json["synonyms"].empty()) {- return Option<bool>(400, "Could not find an array of `synonyms`");- }-- for(const auto& synonym: synonym_json["synonyms"]) {- if(!synonym.is_string() || synonym == "") {- return Option<bool>(400, "Could not find a valid string array of `synonyms`");- }-- std::vector<std::string> tokens;- StringUtils::split(synonym, tokens, " ");- for(auto& token: tokens) {- StringUtils::tolowercase(token);- }- syn.synonyms.push_back(tokens);- }-- if(synonym_json.count("root") != 0) {- std::vector<std::string> tokens;- StringUtils::split(synonym_json["root"], tokens, " ");- for(auto& token: tokens) {- StringUtils::tolowercase(token);- }- syn.root = tokens;- }-- syn.id = synonym_json["id"];- return Option<bool>(true);- }+ static Option<bool> parse(const nlohmann::json& synonym_json, synonym_t& syn);static uint64_t get_hash(const std::vector<std::string>& tokens) {uint64_t hash = 1;diff --git a/src/synonym_index.cpp b/src/synonym_index.cppindex 24b96f4e..eeb40e43 100644--- a/src/synonym_index.cpp+++ b/src/synonym_index.cpp@@ -184,3 +184,87 @@ spp::sparse_hash_map<std::string, synonym_t> SynonymIndex::get_synonyms() {std::string SynonymIndex::get_synonym_key(const std::string & collection_name, const std::string & synonym_id) {return std::string(COLLECTION_SYNONYM_PREFIX) + "_" + collection_name + "_" + synonym_id;}++Option<bool> synonym_t::parse(const nlohmann::json& synonym_json, synonym_t& syn) {+ if(synonym_json.count("id") == 0) {+ return Option<bool>(400, "Missing `id` field.");+ }++ if(synonym_json.count("synonyms") == 0) {+ return Option<bool>(400, "Could not find an array of `synonyms`");+ }++ if(synonym_json.count("root") != 0 && !synonym_json["root"].is_string()) {+ return Option<bool>(400, "Key `root` should be a string.");+ }++ if (!synonym_json["synonyms"].is_array() || synonym_json["synonyms"].empty()) {+ return Option<bool>(400, "Could not find an array of `synonyms`");+ }++ if(synonym_json.count("locale") != 0) {+ if(!synonym_json["locale"].is_string()) {+ return Option<bool>(400, "Synonym `locale` should be a string.`");+ }++ syn.locale = synonym_json["locale"].get<std::string>();+ }++ if(synonym_json.count("symbols_to_index") != 0) {+ if(!synonym_json["symbols_to_index"].is_array() || synonym_json["symbols_to_index"].empty() ||+ !synonym_json["symbols_to_index"][0].is_string()) {+ return Option<bool>(400, "Synonym `symbols_to_index` should be an array of strings.");+ }++ auto symbols = synonym_json["symbols_to_index"].get<std::vector<std::string>>();+ for(auto symbol: symbols) {+ if(symbol.size() != 1) {+ return Option<bool>(400, "Synonym `symbols_to_index` should be an array of single character symbols.");+ }++ syn.symbols.push_back(symbol[0]);+ }+ }++ if(synonym_json.count("root") != 0) {+ std::vector<std::string> tokens;+ Tokenizer(synonym_json["root"], true, false, syn.locale, syn.symbols).tokenize(tokens);+ syn.root = tokens;+ }++ for(const auto& synonym: synonym_json["synonyms"]) {+ if(!synonym.is_string() || synonym == "") {+ return Option<bool>(400, "Could not find a valid string array of `synonyms`");+ }++ std::vector<std::string> tokens;+ Tokenizer(synonym, true, false, syn.locale, syn.symbols).tokenize(tokens);+ syn.synonyms.push_back(tokens);+ }++ syn.id = synonym_json["id"];+ return Option<bool>(true);+}++nlohmann::json synonym_t::to_view_json() const {+ nlohmann::json obj;+ obj["id"] = id;+ obj["root"] = StringUtils::join(root, " ");++ obj["synonyms"] = nlohmann::json::array();++ for(const auto& synonym: synonyms) {+ obj["synonyms"].push_back(StringUtils::join(synonym, " "));+ }++ if(!locale.empty()) {+ obj["locale"] = locale;+ }++ if(!symbols.empty()) {+ obj["symbols"] = symbols;+ }++ return obj;+}+diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 50086f2d..cf49f4f9 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -83,7 +83,8 @@ TEST_F(CollectionSynonymsTest, SynonymParsingFromJson) {nlohmann::json syn_plus_json = {{"id", "syn-plus"},{"root", "+"},- {"synonyms", {"plus", "#"} }+ {"synonyms", {"plus", "#"} },+ {"symbols_to_index", {"+", "#"}},};synonym_t synonym_plus;@@ -169,6 +170,29 @@ TEST_F(CollectionSynonymsTest, SynonymParsingFromJson) {syn_op = synonym_t::parse(syn_json_root_bad_type, synonym);ASSERT_FALSE(syn_op.ok());ASSERT_STREQ("Key `root` should be a string.", syn_op.error().c_str());++ // bad symbols to index+ nlohmann::json syn_json_bad_symbols = {+ {"id", "syn-1"},+ {"root", "Ocean"},+ {"synonyms", {"Sea"} },+ {"symbols_to_index", {}}+ };++ syn_op = synonym_t::parse(syn_json_bad_symbols, synonym);+ ASSERT_FALSE(syn_op.ok());+ ASSERT_STREQ("Synonym `symbols_to_index` should be an array of strings.", syn_op.error().c_str());++ syn_json_bad_symbols = {+ {"id", "syn-1"},+ {"root", "Ocean"},+ {"synonyms", {"Sea"} },+ {"symbols_to_index", {"%^"}}+ };++ syn_op = synonym_t::parse(syn_json_bad_symbols, synonym);+ ASSERT_FALSE(syn_op.ok());+ ASSERT_STREQ("Synonym `symbols_to_index` should be an array of single character symbols.", syn_op.error().c_str());}TEST_F(CollectionSynonymsTest, SynonymReductionOneWay) {@@ -510,8 +534,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {ASSERT_EQ(2, res["hits"].size());ASSERT_EQ(2, res["found"].get<uint32_t>());- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L</mark>. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());// for now we don't support synonyms on ANY prefix@@ -889,7 +913,8 @@ TEST_F(CollectionSynonymsTest, HandleSpecialSymbols) {nlohmann::json syn_plus_json = {{"id", "syn-1"},{"root", "plus"},- {"synonyms", {"+"} }+ {"synonyms", {"+"} },+ {"symbols_to_index", {"+"}}};ASSERT_TRUE(coll1->add_synonym(syn_plus_json).ok());@@ -936,4 +961,53 @@ TEST_F(CollectionSynonymsTest, SynonymForNonAsciiLanguage) {auto res = coll1->search("எல்லோருக்கும்", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();ASSERT_EQ(1, res["hits"].size());ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());-}\ No newline at end of file+}++TEST_F(CollectionSynonymsTest, SynonymForKorean) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string", "locale": "ko"},+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ std::vector<std::vector<std::string>> records = {+ {"도쿄구울", "100"},+ {"도쿄 구울", "100"},+ {"구울", "100"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["points"] = std::stoi(records[i][1]);++ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());+ }++ nlohmann::json synonym1 = R"({+ "id": "syn-1",+ "root": "",+ "synonyms": ["도쿄구울", "도쿄 구울", "구울"],+ "locale": "ko"+ })"_json;++ ASSERT_TRUE(coll1->add_synonym(synonym1).ok());++ auto res = coll1->search("도쿄구울", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(3, res["hits"].size());++ res = coll1->search("도쿄 구울", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(3, res["hits"].size());++ res = coll1->search("구울", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(3, res["hits"].size());+}