Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add boolean to enable synonyms (#1603)

Krunal Gandhid7f638059e52
diff --git a/include/collection.h b/include/collection.h
index a14c00de..3e0cd347 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -584,7 +584,8 @@ public:
std::string conversation_id = "",
const std::string& override_tags_str = "",
const std::string& voice_query = "",
- bool enable_typos_for_numerical_tokens = true) const;
+ bool enable_typos_for_numerical_tokens = true,
+ bool enable_synonyms = true) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --git a/include/index.h b/include/index.h
index 1c8b7460..d719b410 100644
--- a/include/index.h
+++ b/include/index.h
@@ -658,7 +658,8 @@ public:
// Public operations
Option<bool> run_search(search_args* search_params, const std::string& collection_name,
- facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens);
+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,
+ bool enable_synonyms);
Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,
const text_match_type_t match_type,
@@ -688,7 +689,8 @@ public:
const std::string& collection_name,
const drop_tokens_param_t drop_tokens_mode,
facet_index_type_t facet_index_type = DETECT,
- bool enable_typos_for_numerical_tokens = true
+ bool enable_typos_for_numerical_tokens = true,
+ bool enable_synonyms = true
) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
diff --git a/src/collection.cpp b/src/collection.cpp
index 123c93c3..34a5be56 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1714,7 +1714,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
std::string conversation_id,
const std::string& override_tags_str,
const std::string& voice_query,
- bool enable_typos_for_numerical_tokens) const {
+ bool enable_typos_for_numerical_tokens,
+ bool enable_synonyms) const {
std::shared_lock lock(mutex);
// setup thread local vars
@@ -2304,7 +2305,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
std::unique_ptr<search_args> search_params_guard(search_params);
- auto search_op = index->run_search(search_params, name, facet_index_type, enable_typos_for_numerical_tokens);
+ auto search_op = index->run_search(search_params, name, facet_index_type,
+ enable_typos_for_numerical_tokens, enable_synonyms);
// filter_tree_root might be updated in Index::static_filter_query_eval.
filter_tree_root_guard.release();
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 7f1a8aa2..e1af018b 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1369,6 +1369,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *HIDDEN_HITS = "hidden_hits";
const char *ENABLE_OVERRIDES = "enable_overrides";
const char *FILTER_CURATED_HITS = "filter_curated_hits";
+ const char *ENABLE_SYNONYMS = "enable_synonyms";
const char *MAX_CANDIDATES = "max_candidates";
@@ -1524,6 +1525,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
bool prioritize_token_position = false;
bool pre_segmented_query = false;
bool enable_overrides = true;
+ bool enable_synonyms = true;
size_t filter_curated_hits_option = 2;
std::string highlight_fields;
bool exhaustive_search = false;
@@ -1608,6 +1610,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},
{GROUP_MISSING_VALUES, &group_missing_values},
{ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},
+ {ENABLE_SYNONYMS, &enable_synonyms},
};
std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {
@@ -1822,7 +1825,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
conversation_id,
override_tags,
voice_query,
- enable_typos_for_numerical_tokens);
+ enable_typos_for_numerical_tokens,
+ enable_synonyms);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --git a/src/index.cpp b/src/index.cpp
index 1e712213..d3f9dfdd 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2211,7 +2211,8 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string
}
Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,
- facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens) {
+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,
+ bool enable_synonyms) {
return search(search_params->field_query_tokens,
search_params->search_fields,
search_params->match_type,
@@ -2252,7 +2253,8 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& co
collection_name,
search_params->drop_tokens_mode,
facet_index_type,
- enable_typos_for_numerical_tokens
+ enable_typos_for_numerical_tokens,
+ enable_synonyms
);
}
@@ -2740,7 +2742,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
const std::string& collection_name,
const drop_tokens_param_t drop_tokens_mode,
facet_index_type_t facet_index_type,
- bool enable_typos_for_numerical_tokens) const {
+ bool enable_typos_for_numerical_tokens,
+ bool enable_synonyms) const {
std::shared_lock lock(mutex);
auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,
@@ -3083,7 +3086,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
for(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {
q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);
}
- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
+
+ if(enable_synonyms) {
+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
+ }
if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {
auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index 6a0b9707..468c7342 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -1066,4 +1066,87 @@ TEST_F(CollectionSynonymsTest, MultipleSynonymSubstitution) {
res = coll2->search("suit man", {"title", "gender"}, "", {},
{}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, res["hits"].size());
+}
+
+TEST_F(CollectionSynonymsTest, EnableSynonymFlag) {
+ nlohmann::json schema = R"({
+ "name": "coll2",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "gender", "type": "string"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll2 = op.get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"Beautiful Blazer", "Male"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["gender"] = records[i][1];
+
+ auto add_op = coll2->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ nlohmann::json synonym1 = R"({
+ "id": "foobar",
+ "synonyms": ["blazer", "suit"]
+ })"_json;
+
+ nlohmann::json synonym2 = R"({
+ "id": "foobar2",
+ "synonyms": ["male", "man"]
+ })"_json;
+
+
+ ASSERT_TRUE(coll2->add_synonym(synonym1).ok());
+ ASSERT_TRUE(coll2->add_synonym(synonym2).ok());
+ bool enable_synonyms = true;
+
+ auto res = coll2->search("suit man", {"title", "gender"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, enable_synonyms).get();
+
+ ASSERT_EQ(1, res["hits"].size());
+
+ enable_synonyms = false;
+ res = coll2->search("suit man", {"title", "gender"}, "", {},
+ {}, {2}, 10, 1,FREQUENCY, {true},
+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "",
+ 30, 4, "", 40,
+ {}, {}, {}, 0,"<mark>",
+ "</mark>", {}, 1000,true,
+ false, true, "", false,
+ 6000*1000, 4, 7, fallback, 4,
+ {off}, INT16_MAX, INT16_MAX,2,
+ 2, false, "", true,
+ 0, max_score, 100, 0, 0,
+ HASH, 30000, 2, "",
+ {},{}, "right_to_left", true,
+ true, false, "", "", "",
+ "", false, enable_synonyms).get();
+
+ ASSERT_EQ(0, res["hits"].size());
}
\ No newline at end of file