Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Synonyms typos (#1620)
Krunal Gandhic3560dee27ca
diff --git a/include/collection.h b/include/collection.hindex 3e0cd347..949f923c 100644--- a/include/collection.h+++ b/include/collection.h@@ -585,7 +585,9 @@ public:const std::string& override_tags_str = "",const std::string& voice_query = "",bool enable_typos_for_numerical_tokens = true,- bool enable_synonyms = true) const;+ bool enable_synonyms = true,+ bool synonym_prefix = false,+ uint32_t synonym_num_typos = 0) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;@@ -628,7 +630,7 @@ public:// synonym operations- spp::sparse_hash_map<std::string, synonym_t> get_synonyms();+ spp::sparse_hash_map<uint32_t, synonym_t> get_synonyms();bool get_synonym(const std::string& id, synonym_t& synonym);@@ -637,7 +639,8 @@ public:Option<bool> remove_synonym(const std::string & id);void synonym_reduction(const std::vector<std::string>& tokens,- std::vector<std::vector<std::string>>& results) const;+ std::vector<std::vector<std::string>>& results,+ bool synonym_prefix = false, uint32_t synonym_num_typos = 0) const;SynonymIndex* get_synonym_index();diff --git a/include/index.h b/include/index.hindex d719b410..1df40901 100644--- a/include/index.h+++ b/include/index.h@@ -659,7 +659,7 @@ public:Option<bool> run_search(search_args* search_params, const std::string& collection_name,facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,- bool enable_synonyms);+ bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos);Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,const text_match_type_t match_type,@@ -690,7 +690,8 @@ public:const drop_tokens_param_t drop_tokens_mode,facet_index_type_t facet_index_type = DETECT,bool enable_typos_for_numerical_tokens = true,- bool enable_synonyms = true+ bool enable_synonyms = true,+ bool synonym_prefix = false, uint32_t synonym_num_typos = 0) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,diff --git a/include/synonym_index.h b/include/synonym_index.hindex 5035a3f9..8c067088 100644--- a/include/synonym_index.h+++ b/include/synonym_index.h@@ -7,6 +7,7 @@#include "option.h"#include "tokenizer.h"#include "store.h"+#include "art.h"struct synonym_t {std::string id;@@ -49,28 +50,39 @@ private:mutable std::shared_mutex mutex;Store* store;- spp::sparse_hash_map<std::string, synonym_t> synonym_definitions;- spp::sparse_hash_map<uint64_t, std::vector<std::string>> synonym_index;+ spp::sparse_hash_map<std::string, uint32_t> synonym_ids_index_map;+ art_tree* synonym_index_tree;+ uint32_t synonym_index = 0;+ spp::sparse_hash_map<uint32_t, synonym_t> synonym_definitions;void synonym_reduction_internal(const std::vector<std::string>& tokens,size_t start_window_size,size_t start_index_pos,- std::set<uint64_t>& processed_syn_hashes,+ std::set<std::string>& processed_tokens,std::vector<std::vector<std::string>>& results,- const std::vector<std::string>& orig_tokens) const;-+ const std::vector<std::string>& orig_tokens,+ bool synonym_prefix, uint32_t synonym_num_typos) const;public:static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";- SynonymIndex(Store* store): store(store) { }+ SynonymIndex(Store* store): store(store) {+ synonym_index_tree = new art_tree;+ art_tree_init(synonym_index_tree);+ }++ ~SynonymIndex() {+ art_tree_destroy(synonym_index_tree);+ delete synonym_index_tree;+ }static std::string get_synonym_key(const std::string & collection_name, const std::string & synonym_id);void synonym_reduction(const std::vector<std::string>& tokens,- std::vector<std::vector<std::string>>& results) const;+ std::vector<std::vector<std::string>>& results,+ bool synonym_prefix, uint32_t synonym_num_typos) const;- spp::sparse_hash_map<std::string, synonym_t> get_synonyms();+ spp::sparse_hash_map<uint32_t, synonym_t> get_synonyms();bool get_synonym(const std::string& id, synonym_t& synonym);diff --git a/src/collection.cpp b/src/collection.cppindex 34a5be56..d9006746 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1715,7 +1715,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const std::string& override_tags_str,const std::string& voice_query,bool enable_typos_for_numerical_tokens,- bool enable_synonyms) const {+ bool enable_synonyms,+ bool synonym_prefix,+ uint32_t synonyms_num_typos) const {+std::shared_lock lock(mutex);// setup thread local vars@@ -1765,6 +1768,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,return Option<nlohmann::json>(400, "Value of `facet_sample_percent` must be less than 100.");}+ if(synonyms_num_typos > 2) {+ return Option<nlohmann::json>(400, "Value of `synonym_num_typos` must not be greater than 2.");+ }+if(raw_group_by_fields.empty()) {group_limit = 0;}@@ -2306,7 +2313,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::unique_ptr<search_args> search_params_guard(search_params);auto search_op = index->run_search(search_params, name, facet_index_type,- enable_typos_for_numerical_tokens, enable_synonyms);+ enable_typos_for_numerical_tokens, enable_synonyms, synonym_prefix,+ synonyms_num_typos);// filter_tree_root might be updated in Index::static_filter_query_eval.filter_tree_root_guard.release();@@ -4704,12 +4712,13 @@ Option<bool> Collection::remove_synonym(const std::string &id) {}void Collection::synonym_reduction(const std::vector<std::string>& tokens,- std::vector<std::vector<std::string>>& results) const {+ std::vector<std::vector<std::string>>& results,+ bool synonym_prefix, uint32_t synonym_num_typos) const {std::shared_lock lock(mutex);- return synonym_index->synonym_reduction(tokens, results);+ return synonym_index->synonym_reduction(tokens, results, synonym_prefix, synonym_num_typos);}-spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {+spp::sparse_hash_map<uint32_t, synonym_t> Collection::get_synonyms() {std::shared_lock lock(mutex);return synonym_index->get_synonyms();}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex e52700ee..5b841d88 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1421,6 +1421,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *ENABLE_TYPOS_FOR_NUMERICAL_TOKENS = "enable_typos_for_numerical_tokens";+ const char *SYNONYM_PREFIX = "synonym_prefix";+ const char *SYNONYM_NUM_TYPOS = "synonym_num_typos";+// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {if(item.key() == "expires_at") {@@ -1529,6 +1532,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rebool pre_segmented_query = false;bool enable_overrides = true;bool enable_synonyms = true;+ bool synonym_prefix = false;+ size_t synonym_num_typos = 0;+size_t filter_curated_hits_option = 2;std::string highlight_fields;bool exhaustive_search = false;@@ -1583,6 +1589,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{FACET_SAMPLE_THRESHOLD, &facet_sample_threshold},{REMOTE_EMBEDDING_TIMEOUT_MS, &remote_embedding_timeout_ms},{REMOTE_EMBEDDING_NUM_TRIES, &remote_embedding_num_tries},+ {SYNONYM_NUM_TYPOS, &synonym_num_typos},};std::unordered_map<std::string, std::string*> str_values = {@@ -1614,6 +1621,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{GROUP_MISSING_VALUES, &group_missing_values},{ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},{ENABLE_SYNONYMS, &enable_synonyms},+ {SYNONYM_PREFIX, &synonym_prefix},};std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {@@ -1829,7 +1837,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reoverride_tags,voice_query,enable_typos_for_numerical_tokens,- enable_synonyms);+ enable_synonyms,+ synonym_prefix,+ synonym_num_typos);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/src/index.cpp b/src/index.cppindex d3f9dfdd..73538169 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2212,7 +2212,7 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::stringOption<bool> Index::run_search(search_args* search_params, const std::string& collection_name,facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,- bool enable_synonyms) {+ bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos) {return search(search_params->field_query_tokens,search_params->search_fields,search_params->match_type,@@ -2254,8 +2254,9 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cosearch_params->drop_tokens_mode,facet_index_type,enable_typos_for_numerical_tokens,- enable_synonyms- );+ enable_synonyms,+ synonym_prefix,+ synonym_num_typos);}void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,@@ -2743,7 +2744,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consconst drop_tokens_param_t drop_tokens_mode,facet_index_type_t facet_index_type,bool enable_typos_for_numerical_tokens,- bool enable_synonyms) const {+ bool enable_synonyms, bool synonym_prefix,+ uint32_t synonym_num_typos) const {+std::shared_lock lock(mutex);auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,@@ -3088,7 +3091,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}if(enable_synonyms) {- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms,+ synonym_prefix, synonym_num_typos);}if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {diff --git a/src/synonym_index.cpp b/src/synonym_index.cppindex 84802067..6e0fa204 100644--- a/src/synonym_index.cpp+++ b/src/synonym_index.cpp@@ -1,83 +1,66 @@#include "synonym_index.h"-+#include "posting.h"void SynonymIndex::synonym_reduction_internal(const std::vector<std::string>& tokens,size_t start_window_size, size_t start_index_pos,- std::set<uint64_t>& processed_syn_hashes,+ std::set<std::string>& processed_tokens,std::vector<std::vector<std::string>>& results,- const std::vector<std::string>& orig_tokens) const {+ const std::vector<std::string>& orig_tokens,+ bool synonym_prefix, uint32_t synonym_num_typos) const {bool recursed = false;for(size_t window_len = start_window_size; window_len > 0; window_len--) {for(size_t start_index = start_index_pos; start_index+window_len-1 < tokens.size(); start_index++) {- std::vector<uint64_t> syn_hashes;- uint64_t syn_hash = 1;-+ std::string merged_tokens_str="";for(size_t i = start_index; i < start_index+window_len; i++) {- uint64_t token_hash = StringUtils::hash_wy(tokens[i].c_str(), tokens[i].size());-- if(i == start_index) {- syn_hash = token_hash;- } else {- syn_hash = StringUtils::hash_combine(syn_hash, token_hash);- }-- syn_hashes.push_back(token_hash);+ merged_tokens_str += tokens[i];+ merged_tokens_str += " ";}+ StringUtils::trim(merged_tokens_str);- const auto& syn_itr = synonym_index.find(syn_hash);-- if(syn_itr != synonym_index.end() && processed_syn_hashes.count(syn_hash) == 0) {- // tokens in this window match a synonym: reconstruct tokens and rerun synonym mapping against matches- const auto& syn_ids = syn_itr->second;-- for(const auto& syn_id: syn_ids) {- const auto &syn_def = synonym_definitions.at(syn_id);+ std::vector<art_leaf*> leaves;+ std::set<std::string> exclude_leaves;+ auto merged_tokens_len = strlen(merged_tokens_str.c_str());+ merged_tokens_len = synonym_prefix ? merged_tokens_len : merged_tokens_len + 1;- for (const auto &syn_def_tokens: syn_def.synonyms) {- std::vector<std::string> new_tokens;+ art_fuzzy_search(synonym_index_tree, (unsigned char*)merged_tokens_str.c_str(), merged_tokens_len, 0, synonym_num_typos,+ 10, FREQUENCY, synonym_prefix, false, "", nullptr, 0, leaves, exclude_leaves);- for (size_t i = 0; i < start_index; i++) {- new_tokens.push_back(tokens[i]);- }-- std::vector<uint64_t> syn_def_hashes;- uint64_t syn_def_hash = 1;-- for (size_t i = 0; i < syn_def_tokens.size(); i++) {- const auto &syn_def_token = syn_def_tokens[i];- new_tokens.push_back(syn_def_token);- uint64_t token_hash = StringUtils::hash_wy(syn_def_token.c_str(),- syn_def_token.size());-- if (i == 0) {- syn_def_hash = token_hash;- } else {- syn_def_hash = StringUtils::hash_combine(syn_def_hash, token_hash);+ if(processed_tokens.count(merged_tokens_str) == 0) {+ // tokens in this window match a synonym: reconstruct tokens and rerun synonym mapping against matches+ for (const auto &leaf: leaves) {+ for (auto index = 0; index < synonym_index; ++index) {+ if (posting_t::contains(leaf->values, index)) {+ const auto &syn_def = synonym_definitions.at(index);++ for (const auto &syn_def_tokens: syn_def.synonyms) {+ std::vector<std::string> new_tokens;++ for (size_t i = 0; i < start_index; i++) {+ new_tokens.push_back(tokens[i]);+ }++ for (size_t i = 0; i < syn_def_tokens.size(); i++) {+ const auto &syn_def_token = syn_def_tokens[i];+ new_tokens.push_back(syn_def_token);+ processed_tokens.emplace(syn_def_token);+ }++ for (size_t i = start_index + window_len; i < tokens.size(); i++) {+ new_tokens.push_back(tokens[i]);+ }++ processed_tokens.emplace(merged_tokens_str);+ auto syn_def_tokens_str = StringUtils::join(syn_def_tokens, " ");+ processed_tokens.emplace(syn_def_tokens_str);++ recursed = true;+ synonym_reduction_internal(new_tokens, window_len,+ start_index, processed_tokens, results, orig_tokens,+ synonym_prefix, synonym_num_typos);}-- syn_def_hashes.push_back(token_hash);- }-- for (size_t i = start_index + window_len; i < tokens.size(); i++) {- new_tokens.push_back(tokens[i]);- }-- processed_syn_hashes.emplace(syn_def_hash);- processed_syn_hashes.emplace(syn_hash);-- for (uint64_t h: syn_def_hashes) {- processed_syn_hashes.emplace(h);}-- for (uint64_t h: syn_hashes) {- processed_syn_hashes.emplace(h);- }-- recursed = true;- synonym_reduction_internal(new_tokens, window_len,- start_index, processed_syn_hashes, results, orig_tokens);}}}@@ -87,47 +70,64 @@ void SynonymIndex::synonym_reduction_internal(const std::vector<std::string>& tostart_index_pos = 0;}- if(!recursed && !processed_syn_hashes.empty() && tokens != orig_tokens) {+ if(!recursed && !processed_tokens.empty() && tokens != orig_tokens) {results.emplace_back(tokens);}}void SynonymIndex::synonym_reduction(const std::vector<std::string>& tokens,- std::vector<std::vector<std::string>>& results) const {+ std::vector<std::vector<std::string>>& results,+ bool synonym_prefix, uint32_t synonym_num_typos) const {std::shared_lock lock(mutex);if(synonym_definitions.empty()) {return;}- std::set<uint64_t> processed_syn_hashes;- synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results, tokens);+ std::set<std::string> processed_tokens;+ synonym_reduction_internal(tokens, tokens.size(), 0, processed_tokens, results, tokens,+ synonym_prefix, synonym_num_typos);}Option<bool> SynonymIndex::add_synonym(const std::string & collection_name, const synonym_t& synonym,bool write_to_store) {std::unique_lock write_lock(mutex);- if(synonym_definitions.count(synonym.id) != 0) {+ if(synonym_ids_index_map.count(synonym.id) != 0) {write_lock.unlock();// first we have to delete existing entries so we can upsertOption<bool> rem_op = remove_synonym(collection_name, synonym.id);- if(!rem_op.ok()) {+ if (!rem_op.ok()) {return rem_op;}- write_lock.lock();+ write_lock.lock();}- synonym_definitions[synonym.id] = synonym;+ synonym_definitions[synonym_index] = synonym;+ synonym_ids_index_map[synonym.id] = synonym_index;++ std::vector<std::string> keys;if(!synonym.root.empty()) {- uint64_t root_hash = synonym_t::get_hash(synonym.root);- synonym_index[root_hash].emplace_back(synonym.id);+ auto root_tokens_str = StringUtils::join(synonym.root, " ");+ keys.push_back(root_tokens_str);} else {for(const auto & syn_tokens : synonym.synonyms) {- uint64_t syn_hash = synonym_t::get_hash(syn_tokens);- synonym_index[syn_hash].emplace_back(synonym.id);+ auto synonyms_str = StringUtils::join(syn_tokens, " ");+ keys.push_back(synonyms_str);}}+ for(const auto& key : keys) {+ art_leaf* exact_leaf = (art_leaf *) art_search(synonym_index_tree, (unsigned char *) key.c_str(), key.size() + 1);+ if(exact_leaf) {+ auto offset = posting_t::num_ids(exact_leaf->values);+ posting_t::upsert(exact_leaf->values, synonym_index, {offset});+ } else {+ art_document document(synonym_index, synonym_index, {0});+ art_insert(synonym_index_tree, (unsigned char *) key.c_str(), key.size() + 1, &document);+ }+ }+ ++synonym_index;+write_lock.unlock();if(write_to_store) {@@ -143,8 +143,9 @@ Option<bool> SynonymIndex::add_synonym(const std::string & collection_name, consbool SynonymIndex::get_synonym(const std::string& id, synonym_t& synonym) {std::shared_lock lock(mutex);- if(synonym_definitions.count(id) != 0) {- synonym = synonym_definitions[id];+ if(synonym_ids_index_map.count(id) != 0) {+ auto index = synonym_ids_index_map.at(id);+ synonym = synonym_definitions.at(index);return true;}@@ -153,33 +154,39 @@ bool SynonymIndex::get_synonym(const std::string& id, synonym_t& synonym) {Option<bool> SynonymIndex::remove_synonym(const std::string & collection_name, const std::string &id) {std::unique_lock lock(mutex);- const auto& syn_iter = synonym_definitions.find(id);+ const auto& syn_iter = synonym_ids_index_map.find(id);- if(syn_iter != synonym_definitions.end()) {+ if(syn_iter != synonym_ids_index_map.end()) {bool removed = store->remove(get_synonym_key(collection_name, id));if(!removed) {return Option<bool>(500, "Error while deleting the synonym from disk.");}- const auto& synonym = syn_iter->second;+ const auto& synonym = synonym_definitions.at(syn_iter->second);+ std::vector<std::string> keys;if(!synonym.root.empty()) {- uint64_t root_hash = synonym_t::get_hash(synonym.root);- synonym_index.erase(root_hash);+ keys.insert(keys.end(), synonym.root.begin(), synonym.root.end());} else {for(const auto & syn_tokens : synonym.synonyms) {- uint64_t syn_hash = synonym_t::get_hash(syn_tokens);- synonym_index.erase(syn_hash);+ keys.insert(keys.end(), syn_tokens.begin(), syn_tokens.end());}}- synonym_definitions.erase(id);+ for(const auto& key : keys) {+ art_delete(synonym_index_tree, (unsigned char*)key.c_str(), key.size() + 1);+ }++ auto index = synonym_ids_index_map.at(id);+ synonym_ids_index_map.erase(id);+ synonym_definitions.erase(index);+return Option<bool>(true);}return Option<bool>(404, "Could not find that `id`.");}-spp::sparse_hash_map<std::string, synonym_t> SynonymIndex::get_synonyms() {+spp::sparse_hash_map<uint32_t, synonym_t> SynonymIndex::get_synonyms() {std::shared_lock lock(mutex);return synonym_definitions;}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 86482919..4cd77d39 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -527,13 +527,13 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {const auto& synonyms = collection1->get_synonyms();ASSERT_EQ(2, synonyms.size());- ASSERT_STREQ("id1", synonyms.at("id1").id.c_str());- ASSERT_EQ(2, synonyms.at("id1").root.size());- ASSERT_EQ(1, synonyms.at("id1").synonyms.size());+ ASSERT_STREQ("id1", synonyms.at(0).id.c_str());+ ASSERT_EQ(2, synonyms.at(0).root.size());+ ASSERT_EQ(1, synonyms.at(0).synonyms.size());- ASSERT_STREQ("id3", synonyms.at("id3").id.c_str());- ASSERT_EQ(0, synonyms.at("id3").root.size());- ASSERT_EQ(2, synonyms.at("id3").synonyms.size());+ ASSERT_STREQ("id3", synonyms.at(1).id.c_str());+ ASSERT_EQ(0, synonyms.at(1).root.size());+ ASSERT_EQ(2, synonyms.at(1).synonyms.size());std::vector<char> expected_symbols = {'+'};std::vector<char> expected_separators = {'-'};diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 468c7342..130d77d5 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -703,8 +703,9 @@ TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) {ASSERT_EQ(2, coll_mul_fields->get_synonyms().size());coll_mul_fields->remove_synonym("ipod-synonyms");- ASSERT_EQ(1, coll_mul_fields->get_synonyms().size());- ASSERT_STREQ("samsung-synonyms", coll_mul_fields->get_synonyms()["samsung-synonyms"].id.c_str());+ auto synonyms = coll_mul_fields->get_synonyms();+ ASSERT_EQ(1, synonyms.size());+ ASSERT_STREQ("samsung-synonyms", synonyms.begin()->second.id.c_str());// try to upsert synonym with same ID@@ -1149,4 +1150,174 @@ TEST_F(CollectionSynonymsTest, EnableSynonymFlag) {"", false, enable_synonyms).get();ASSERT_EQ(0, res["hits"].size());+}++TEST_F(CollectionSynonymsTest, SynonymTypos) {+ nlohmann::json schema = R"({+ "name": "coll3",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll3 = op.get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "Cool Trousers";++ auto add_op = coll3->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ nlohmann::json synonym1 = R"({+ "id": "foobar",+ "synonyms": ["trousers", "pants"]+ })"_json;++ ASSERT_TRUE(coll3->add_synonym(synonym1).ok());++ auto res = coll3->search("trousers", {"title"}, "", {},+ {}, {0}, 10, 1, FREQUENCY, {true},0).get();+ ASSERT_EQ(1, res["hits"].size());++ res = coll3->search("pants", {"title"}, "", {},+ {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(1, res["hits"].size());++ //try with typos+ uint32_t synonym_num_typos = 0;+ res = coll3->search("patns", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, true, false, synonym_num_typos).get();+ ASSERT_EQ(0, res["hits"].size());++ synonym_num_typos = 2;++ res = coll3->search("patns", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, true, false, synonym_num_typos).get();+ ASSERT_EQ(1, res["hits"].size());++ //max 2 typos supported+ synonym_num_typos = 3;+ auto search_op = coll3->search("trosuers", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, true, false, synonym_num_typos);++ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("Value of `synonym_num_typos` must not be greater than 2.",search_op.error());+}++TEST_F(CollectionSynonymsTest, SynonymPrefix) {+ nlohmann::json schema = R"({+ "name": "coll3",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll3 = op.get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "Cool Trousers";+ auto add_op = coll3->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ doc["id"] = "1";+ doc["title"] = "Cool Pants";+ add_op = coll3->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ nlohmann::json synonym1 = R"({+ "id": "foobar",+ "synonyms": ["trousers", "pants"]+ })"_json;++ ASSERT_TRUE(coll3->add_synonym(synonym1).ok());++ bool synonym_prefix = false;++ auto res = coll3->search("pan", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {false},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, true, synonym_prefix).get();++ ASSERT_EQ(0, res["hits"].size());++ synonym_prefix = true;++ res = coll3->search("pan", {"title"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {false},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, true, synonym_prefix).get();++ ASSERT_EQ(2, res["hits"].size());}\ No newline at end of file