Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Synonyms typos (#1620)

Krunal Gandhic3560dee27ca
diff --git a/include/collection.h b/include/collection.h
index 3e0cd347..949f923c 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -585,7 +585,9 @@ public:
const std::string& override_tags_str = "",
const std::string& voice_query = "",
bool enable_typos_for_numerical_tokens = true,
- bool enable_synonyms = true) const;
+ bool enable_synonyms = true,
+ bool synonym_prefix = false,
+ uint32_t synonym_num_typos = 0) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
@@ -628,7 +630,7 @@ public:
// synonym operations
- spp::sparse_hash_map<std::string, synonym_t> get_synonyms();
+ spp::sparse_hash_map<uint32_t, synonym_t> get_synonyms();
bool get_synonym(const std::string& id, synonym_t& synonym);
@@ -637,7 +639,8 @@ public:
Option<bool> remove_synonym(const std::string & id);
void synonym_reduction(const std::vector<std::string>& tokens,
- std::vector<std::vector<std::string>>& results) const;
+ std::vector<std::vector<std::string>>& results,
+ bool synonym_prefix = false, uint32_t synonym_num_typos = 0) const;
SynonymIndex* get_synonym_index();
diff --git a/include/index.h b/include/index.h
index d719b410..1df40901 100644
--- a/include/index.h
+++ b/include/index.h
@@ -659,7 +659,7 @@ public:
Option<bool> run_search(search_args* search_params, const std::string& collection_name,
facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,
- bool enable_synonyms);
+ bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos);
Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
@@ -690,7 +690,8 @@ public:
const drop_tokens_param_t drop_tokens_mode,
facet_index_type_t facet_index_type = DETECT,
bool enable_typos_for_numerical_tokens = true,
- bool enable_synonyms = true
+ bool enable_synonyms = true,
+ bool synonym_prefix = false, uint32_t synonym_num_typos = 0
) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
diff --git a/include/synonym_index.h b/include/synonym_index.h
index 5035a3f9..8c067088 100644
--- a/include/synonym_index.h
+++ b/include/synonym_index.h
@@ -7,6 +7,7 @@
#include "option.h"
#include "tokenizer.h"
#include "store.h"
+#include "art.h"
struct synonym_t {
std::string id;
@@ -49,28 +50,39 @@ private:
mutable std::shared_mutex mutex;
Store* store;
- spp::sparse_hash_map<std::string, synonym_t> synonym_definitions;
- spp::sparse_hash_map<uint64_t, std::vector<std::string>> synonym_index;
+ spp::sparse_hash_map<std::string, uint32_t> synonym_ids_index_map;
+ art_tree* synonym_index_tree;
+ uint32_t synonym_index = 0;
+ spp::sparse_hash_map<uint32_t, synonym_t> synonym_definitions;
void synonym_reduction_internal(const std::vector<std::string>& tokens,
size_t start_window_size,
size_t start_index_pos,
- std::set<uint64_t>& processed_syn_hashes,
+ std::set<std::string>& processed_tokens,
std::vector<std::vector<std::string>>& results,
- const std::vector<std::string>& orig_tokens) const;
-
+ const std::vector<std::string>& orig_tokens,
+ bool synonym_prefix, uint32_t synonym_num_typos) const;
public:
static constexpr const char* COLLECTION_SYNONYM_PREFIX = "$CY";
- SynonymIndex(Store* store): store(store) { }
+ SynonymIndex(Store* store): store(store) {
+ synonym_index_tree = new art_tree;
+ art_tree_init(synonym_index_tree);
+ }
+
+ ~SynonymIndex() {
+ art_tree_destroy(synonym_index_tree);
+ delete synonym_index_tree;
+ }
static std::string get_synonym_key(const std::string & collection_name, const std::string & synonym_id);
void synonym_reduction(const std::vector<std::string>& tokens,
- std::vector<std::vector<std::string>>& results) const;
+ std::vector<std::vector<std::string>>& results,
+ bool synonym_prefix, uint32_t synonym_num_typos) const;
- spp::sparse_hash_map<std::string, synonym_t> get_synonyms();
+ spp::sparse_hash_map<uint32_t, synonym_t> get_synonyms();
bool get_synonym(const std::string& id, synonym_t& synonym);
diff --git a/src/collection.cpp b/src/collection.cpp
index 34a5be56..d9006746 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1715,7 +1715,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const std::string& override_tags_str,
const std::string& voice_query,
bool enable_typos_for_numerical_tokens,
- bool enable_synonyms) const {
+ bool enable_synonyms,
+ bool synonym_prefix,
+ uint32_t synonyms_num_typos) const {
+
std::shared_lock lock(mutex);
// setup thread local vars
@@ -1765,6 +1768,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
return Option<nlohmann::json>(400, "Value of `facet_sample_percent` must be less than 100.");
}
+ if(synonyms_num_typos > 2) {
+ return Option<nlohmann::json>(400, "Value of `synonym_num_typos` must not be greater than 2.");
+ }
+
if(raw_group_by_fields.empty()) {
group_limit = 0;
}
@@ -2306,7 +2313,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
std::unique_ptr<search_args> search_params_guard(search_params);
auto search_op = index->run_search(search_params, name, facet_index_type,
- enable_typos_for_numerical_tokens, enable_synonyms);
+ enable_typos_for_numerical_tokens, enable_synonyms, synonym_prefix,
+ synonyms_num_typos);
// filter_tree_root might be updated in Index::static_filter_query_eval.
filter_tree_root_guard.release();
@@ -4704,12 +4712,13 @@ Option<bool> Collection::remove_synonym(const std::string &id) {
}
void Collection::synonym_reduction(const std::vector<std::string>& tokens,
- std::vector<std::vector<std::string>>& results) const {
+ std::vector<std::vector<std::string>>& results,
+ bool synonym_prefix, uint32_t synonym_num_typos) const {
std::shared_lock lock(mutex);
- return synonym_index->synonym_reduction(tokens, results);
+ return synonym_index->synonym_reduction(tokens, results, synonym_prefix, synonym_num_typos);
}
-spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {
+spp::sparse_hash_map<uint32_t, synonym_t> Collection::get_synonyms() {
std::shared_lock lock(mutex);
return synonym_index->get_synonyms();
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index e52700ee..5b841d88 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1421,6 +1421,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *ENABLE_TYPOS_FOR_NUMERICAL_TOKENS = "enable_typos_for_numerical_tokens";
+ const char *SYNONYM_PREFIX = "synonym_prefix";
+ const char *SYNONYM_NUM_TYPOS = "synonym_num_typos";
+
// enrich params with values from embedded params
for(auto& item: embedded_params.items()) {
if(item.key() == "expires_at") {
@@ -1529,6 +1532,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
bool pre_segmented_query = false;
bool enable_overrides = true;
bool enable_synonyms = true;
+ bool synonym_prefix = false;
+ size_t synonym_num_typos = 0;
+
size_t filter_curated_hits_option = 2;
std::string highlight_fields;
bool exhaustive_search = false;
@@ -1583,6 +1589,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{FACET_SAMPLE_THRESHOLD, &facet_sample_threshold},
{REMOTE_EMBEDDING_TIMEOUT_MS, &remote_embedding_timeout_ms},
{REMOTE_EMBEDDING_NUM_TRIES, &remote_embedding_num_tries},
+ {SYNONYM_NUM_TYPOS, &synonym_num_typos},
};
std::unordered_map<std::string, std::string*> str_values = {
@@ -1614,6 +1621,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{GROUP_MISSING_VALUES, &group_missing_values},
{ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},
{ENABLE_SYNONYMS, &enable_synonyms},
+ {SYNONYM_PREFIX, &synonym_prefix},
};
std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {
@@ -1829,7 +1837,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
override_tags,
voice_query,
enable_typos_for_numerical_tokens,
- enable_synonyms);
+ enable_synonyms,
+ synonym_prefix,
+ synonym_num_typos);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/src/index.cpp b/src/index.cpp
index d3f9dfdd..73538169 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2212,7 +2212,7 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string
Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,
facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,
- bool enable_synonyms) {
+ bool enable_synonyms, bool synonym_prefix, uint32_t synonym_num_typos) {
return search(search_params->field_query_tokens,
search_params->search_fields,
search_params->match_type,
@@ -2254,8 +2254,9 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& co
search_params->drop_tokens_mode,
facet_index_type,
enable_typos_for_numerical_tokens,
- enable_synonyms
- );
+ enable_synonyms,
+ synonym_prefix,
+ synonym_num_typos);
}
void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
@@ -2743,7 +2744,9 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
const drop_tokens_param_t drop_tokens_mode,
facet_index_type_t facet_index_type,
bool enable_typos_for_numerical_tokens,
- bool enable_synonyms) const {
+ bool enable_synonyms, bool synonym_prefix,
+ uint32_t synonym_num_typos) const {
+
std::shared_lock lock(mutex);
auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,
@@ -3088,7 +3091,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
if(enable_synonyms) {
- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms,
+ synonym_prefix, synonym_num_typos);
}
if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {
diff --git a/src/synonym_index.cpp b/src/synonym_index.cpp
index 84802067..6e0fa204 100644
--- a/src/synonym_index.cpp
+++ b/src/synonym_index.cpp
@@ -1,83 +1,66 @@
#include "synonym_index.h"
-
+#include "posting.h"
void SynonymIndex::synonym_reduction_internal(const std::vector<std::string>& tokens,
size_t start_window_size, size_t start_index_pos,
- std::set<uint64_t>& processed_syn_hashes,
+ std::set<std::string>& processed_tokens,
std::vector<std::vector<std::string>>& results,
- const std::vector<std::string>& orig_tokens) const {
+ const std::vector<std::string>& orig_tokens,
+ bool synonym_prefix, uint32_t synonym_num_typos) const {
bool recursed = false;
for(size_t window_len = start_window_size; window_len > 0; window_len--) {
for(size_t start_index = start_index_pos; start_index+window_len-1 < tokens.size(); start_index++) {
- std::vector<uint64_t> syn_hashes;
- uint64_t syn_hash = 1;
-
+ std::string merged_tokens_str="";
for(size_t i = start_index; i < start_index+window_len; i++) {
- uint64_t token_hash = StringUtils::hash_wy(tokens[i].c_str(), tokens[i].size());
-
- if(i == start_index) {
- syn_hash = token_hash;
- } else {
- syn_hash = StringUtils::hash_combine(syn_hash, token_hash);
- }
-
- syn_hashes.push_back(token_hash);
+ merged_tokens_str += tokens[i];
+ merged_tokens_str += " ";
}
+ StringUtils::trim(merged_tokens_str);
- const auto& syn_itr = synonym_index.find(syn_hash);
-
- if(syn_itr != synonym_index.end() && processed_syn_hashes.count(syn_hash) == 0) {
- // tokens in this window match a synonym: reconstruct tokens and rerun synonym mapping against matches
- const auto& syn_ids = syn_itr->second;
-
- for(const auto& syn_id: syn_ids) {
- const auto &syn_def = synonym_definitions.at(syn_id);
+ std::vector<art_leaf*> leaves;
+ std::set<std::string> exclude_leaves;
+ auto merged_tokens_len = strlen(merged_tokens_str.c_str());
+ merged_tokens_len = synonym_prefix ? merged_tokens_len : merged_tokens_len + 1;
- for (const auto &syn_def_tokens: syn_def.synonyms) {
- std::vector<std::string> new_tokens;
+ art_fuzzy_search(synonym_index_tree, (unsigned char*)merged_tokens_str.c_str(), merged_tokens_len, 0, synonym_num_typos,
+ 10, FREQUENCY, synonym_prefix, false, "", nullptr, 0, leaves, exclude_leaves);
- for (size_t i = 0; i < start_index; i++) {
- new_tokens.push_back(tokens[i]);
- }
-
- std::vector<uint64_t> syn_def_hashes;
- uint64_t syn_def_hash = 1;
-
- for (size_t i = 0; i < syn_def_tokens.size(); i++) {
- const auto &syn_def_token = syn_def_tokens[i];
- new_tokens.push_back(syn_def_token);
- uint64_t token_hash = StringUtils::hash_wy(syn_def_token.c_str(),
- syn_def_token.size());
-
- if (i == 0) {
- syn_def_hash = token_hash;
- } else {
- syn_def_hash = StringUtils::hash_combine(syn_def_hash, token_hash);
+ if(processed_tokens.count(merged_tokens_str) == 0) {
+ // tokens in this window match a synonym: reconstruct tokens and rerun synonym mapping against matches
+ for (const auto &leaf: leaves) {
+ for (auto index = 0; index < synonym_index; ++index) {
+ if (posting_t::contains(leaf->values, index)) {
+ const auto &syn_def = synonym_definitions.at(index);
+
+ for (const auto &syn_def_tokens: syn_def.synonyms) {
+ std::vector<std::string> new_tokens;
+
+ for (size_t i = 0; i < start_index; i++) {
+ new_tokens.push_back(tokens[i]);
+ }
+
+ for (size_t i = 0; i < syn_def_tokens.size(); i++) {
+ const auto &syn_def_token = syn_def_tokens[i];
+ new_tokens.push_back(syn_def_token);
+ processed_tokens.emplace(syn_def_token);
+ }
+
+ for (size_t i = start_index + window_len; i < tokens.size(); i++) {
+ new_tokens.push_back(tokens[i]);
+ }
+
+ processed_tokens.emplace(merged_tokens_str);
+ auto syn_def_tokens_str = StringUtils::join(syn_def_tokens, " ");
+ processed_tokens.emplace(syn_def_tokens_str);
+
+ recursed = true;
+ synonym_reduction_internal(new_tokens, window_len,
+ start_index, processed_tokens, results, orig_tokens,
+ synonym_prefix, synonym_num_typos);
}
-
- syn_def_hashes.push_back(token_hash);
- }
-
- for (size_t i = start_index + window_len; i < tokens.size(); i++) {
- new_tokens.push_back(tokens[i]);
- }
-
- processed_syn_hashes.emplace(syn_def_hash);
- processed_syn_hashes.emplace(syn_hash);
-
- for (uint64_t h: syn_def_hashes) {
- processed_syn_hashes.emplace(h);
}
-
- for (uint64_t h: syn_hashes) {
- processed_syn_hashes.emplace(h);
- }
-
- recursed = true;
- synonym_reduction_internal(new_tokens, window_len,
- start_index, processed_syn_hashes, results, orig_tokens);
}
}
}
@@ -87,47 +70,64 @@ void SynonymIndex::synonym_reduction_internal(const std::vector<std::string>& to
start_index_pos = 0;
}
- if(!recursed && !processed_syn_hashes.empty() && tokens != orig_tokens) {
+ if(!recursed && !processed_tokens.empty() && tokens != orig_tokens) {
results.emplace_back(tokens);
}
}
void SynonymIndex::synonym_reduction(const std::vector<std::string>& tokens,
- std::vector<std::vector<std::string>>& results) const {
+ std::vector<std::vector<std::string>>& results,
+ bool synonym_prefix, uint32_t synonym_num_typos) const {
std::shared_lock lock(mutex);
if(synonym_definitions.empty()) {
return;
}
- std::set<uint64_t> processed_syn_hashes;
- synonym_reduction_internal(tokens, tokens.size(), 0, processed_syn_hashes, results, tokens);
+ std::set<std::string> processed_tokens;
+ synonym_reduction_internal(tokens, tokens.size(), 0, processed_tokens, results, tokens,
+ synonym_prefix, synonym_num_typos);
}
Option<bool> SynonymIndex::add_synonym(const std::string & collection_name, const synonym_t& synonym,
bool write_to_store) {
std::unique_lock write_lock(mutex);
- if(synonym_definitions.count(synonym.id) != 0) {
+ if(synonym_ids_index_map.count(synonym.id) != 0) {
write_lock.unlock();
// first we have to delete existing entries so we can upsert
Option<bool> rem_op = remove_synonym(collection_name, synonym.id);
- if(!rem_op.ok()) {
+ if (!rem_op.ok()) {
return rem_op;
}
- write_lock.lock();
+ write_lock.lock();
}
- synonym_definitions[synonym.id] = synonym;
+ synonym_definitions[synonym_index] = synonym;
+ synonym_ids_index_map[synonym.id] = synonym_index;
+
+ std::vector<std::string> keys;
if(!synonym.root.empty()) {
- uint64_t root_hash = synonym_t::get_hash(synonym.root);
- synonym_index[root_hash].emplace_back(synonym.id);
+ auto root_tokens_str = StringUtils::join(synonym.root, " ");
+ keys.push_back(root_tokens_str);
} else {
for(const auto & syn_tokens : synonym.synonyms) {
- uint64_t syn_hash = synonym_t::get_hash(syn_tokens);
- synonym_index[syn_hash].emplace_back(synonym.id);
+ auto synonyms_str = StringUtils::join(syn_tokens, " ");
+ keys.push_back(synonyms_str);
}
}
+ for(const auto& key : keys) {
+ art_leaf* exact_leaf = (art_leaf *) art_search(synonym_index_tree, (unsigned char *) key.c_str(), key.size() + 1);
+ if(exact_leaf) {
+ auto offset = posting_t::num_ids(exact_leaf->values);
+ posting_t::upsert(exact_leaf->values, synonym_index, {offset});
+ } else {
+ art_document document(synonym_index, synonym_index, {0});
+ art_insert(synonym_index_tree, (unsigned char *) key.c_str(), key.size() + 1, &document);
+ }
+ }
+ ++synonym_index;
+
write_lock.unlock();
if(write_to_store) {
@@ -143,8 +143,9 @@ Option<bool> SynonymIndex::add_synonym(const std::string & collection_name, cons
bool SynonymIndex::get_synonym(const std::string& id, synonym_t& synonym) {
std::shared_lock lock(mutex);
- if(synonym_definitions.count(id) != 0) {
- synonym = synonym_definitions[id];
+ if(synonym_ids_index_map.count(id) != 0) {
+ auto index = synonym_ids_index_map.at(id);
+ synonym = synonym_definitions.at(index);
return true;
}
@@ -153,33 +154,39 @@ bool SynonymIndex::get_synonym(const std::string& id, synonym_t& synonym) {
Option<bool> SynonymIndex::remove_synonym(const std::string & collection_name, const std::string &id) {
std::unique_lock lock(mutex);
- const auto& syn_iter = synonym_definitions.find(id);
+ const auto& syn_iter = synonym_ids_index_map.find(id);
- if(syn_iter != synonym_definitions.end()) {
+ if(syn_iter != synonym_ids_index_map.end()) {
bool removed = store->remove(get_synonym_key(collection_name, id));
if(!removed) {
return Option<bool>(500, "Error while deleting the synonym from disk.");
}
- const auto& synonym = syn_iter->second;
+ const auto& synonym = synonym_definitions.at(syn_iter->second);
+ std::vector<std::string> keys;
if(!synonym.root.empty()) {
- uint64_t root_hash = synonym_t::get_hash(synonym.root);
- synonym_index.erase(root_hash);
+ keys.insert(keys.end(), synonym.root.begin(), synonym.root.end());
} else {
for(const auto & syn_tokens : synonym.synonyms) {
- uint64_t syn_hash = synonym_t::get_hash(syn_tokens);
- synonym_index.erase(syn_hash);
+ keys.insert(keys.end(), syn_tokens.begin(), syn_tokens.end());
}
}
- synonym_definitions.erase(id);
+ for(const auto& key : keys) {
+ art_delete(synonym_index_tree, (unsigned char*)key.c_str(), key.size() + 1);
+ }
+
+ auto index = synonym_ids_index_map.at(id);
+ synonym_ids_index_map.erase(id);
+ synonym_definitions.erase(index);
+
return Option<bool>(true);
}
return Option<bool>(404, "Could not find that `id`.");
}
-spp::sparse_hash_map<std::string, synonym_t> SynonymIndex::get_synonyms() {
+spp::sparse_hash_map<uint32_t, synonym_t> SynonymIndex::get_synonyms() {
std::shared_lock lock(mutex);
return synonym_definitions;
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 86482919..4cd77d39 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -527,13 +527,13 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
const auto& synonyms = collection1->get_synonyms();
ASSERT_EQ(2, synonyms.size());
- ASSERT_STREQ("id1", synonyms.at("id1").id.c_str());
- ASSERT_EQ(2, synonyms.at("id1").root.size());
- ASSERT_EQ(1, synonyms.at("id1").synonyms.size());
+ ASSERT_STREQ("id1", synonyms.at(0).id.c_str());
+ ASSERT_EQ(2, synonyms.at(0).root.size());
+ ASSERT_EQ(1, synonyms.at(0).synonyms.size());
- ASSERT_STREQ("id3", synonyms.at("id3").id.c_str());
- ASSERT_EQ(0, synonyms.at("id3").root.size());
- ASSERT_EQ(2, synonyms.at("id3").synonyms.size());
+ ASSERT_STREQ("id3", synonyms.at(1).id.c_str());
+ ASSERT_EQ(0, synonyms.at(1).root.size());
+ ASSERT_EQ(2, synonyms.at(1).synonyms.size());
std::vector<char> expected_symbols = {'+'};
std::vector<char> expected_separators = {'-'};
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index 468c7342..130d77d5 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -703,8 +703,9 @@ TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) {
ASSERT_EQ(2, coll_mul_fields->get_synonyms().size());
coll_mul_fields->remove_synonym("ipod-synonyms");
- ASSERT_EQ(1, coll_mul_fields->get_synonyms().size());
- ASSERT_STREQ("samsung-synonyms", coll_mul_fields->get_synonyms()["samsung-synonyms"].id.c_str());
+ auto synonyms = coll_mul_fields->get_synonyms();
+ ASSERT_EQ(1, synonyms.size());
+ ASSERT_STREQ("samsung-synonyms", synonyms.begin()->second.id.c_str());
// try to upsert synonym with same ID
@@ -1149,4 +1150,174 @@ TEST_F(CollectionSynonymsTest, EnableSynonymFlag) {
"", false, enable_synonyms).get();
ASSERT_EQ(0, res["hits"].size());
+}
+
+TEST_F(CollectionSynonymsTest, SynonymTypos) {
+ nlohmann::json schema = R"({
+ "name": "coll3",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll3 = op.get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "Cool Trousers";
+
+ auto add_op = coll3->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json synonym1 = R"({
+ "id": "foobar",
+ "synonyms": ["trousers", "pants"]
+ })"_json;
+
+ ASSERT_TRUE(coll3->add_synonym(synonym1).ok());
+
+ auto res = coll3->search("trousers", {"title"}, "", {},
+ {}, {0}, 10, 1, FREQUENCY, {true},0).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ res = coll3->search("pants", {"title"}, "", {},
+ {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ //try with typos
+ uint32_t synonym_num_typos = 0;
+ res = coll3->search("patns", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, false, synonym_num_typos).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ synonym_num_typos = 2;
+
+ res = coll3->search("patns", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, false, synonym_num_typos).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ //max 2 typos supported
+ synonym_num_typos = 3;
+ auto search_op = coll3->search("trosuers", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, false, synonym_num_typos);
+
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("Value of `synonym_num_typos` must not be greater than 2.",search_op.error());
+}
+
+TEST_F(CollectionSynonymsTest, SynonymPrefix) {
+ nlohmann::json schema = R"({
+ "name": "coll3",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection *coll3 = op.get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "Cool Trousers";
+ auto add_op = coll3->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ doc["id"] = "1";
+ doc["title"] = "Cool Pants";
+ add_op = coll3->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ nlohmann::json synonym1 = R"({
+ "id": "foobar",
+ "synonyms": ["trousers", "pants"]
+ })"_json;
+
+ ASSERT_TRUE(coll3->add_synonym(synonym1).ok());
+
+ bool synonym_prefix = false;
+
+ auto res = coll3->search("pan", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {false},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, synonym_prefix).get();
+
+ ASSERT_EQ(0, res["hits"].size());
+
+ synonym_prefix = true;
+
+ res = coll3->search("pan", {"title"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {false},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, true, synonym_prefix).get();
+
+ ASSERT_EQ(2, res["hits"].size());
}
\ No newline at end of file