Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
add boolean to enable synonyms (#1603)
Krunal Gandhid7f638059e52
diff --git a/include/collection.h b/include/collection.hindex a14c00de..3e0cd347 100644--- a/include/collection.h+++ b/include/collection.h@@ -584,7 +584,8 @@ public:std::string conversation_id = "",const std::string& override_tags_str = "",const std::string& voice_query = "",- bool enable_typos_for_numerical_tokens = true) const;+ bool enable_typos_for_numerical_tokens = true,+ bool enable_synonyms = true) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/index.h b/include/index.hindex 1c8b7460..d719b410 100644--- a/include/index.h+++ b/include/index.h@@ -658,7 +658,8 @@ public:// Public operationsOption<bool> run_search(search_args* search_params, const std::string& collection_name,- facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens);+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,+ bool enable_synonyms);Option<bool> search(std::vector<query_tokens_t>& field_query_tokens, const std::vector<search_field_t>& the_fields,const text_match_type_t match_type,@@ -688,7 +689,8 @@ public:const std::string& collection_name,const drop_tokens_param_t drop_tokens_mode,facet_index_type_t facet_index_type = DETECT,- bool enable_typos_for_numerical_tokens = true+ bool enable_typos_for_numerical_tokens = true,+ bool enable_synonyms = true) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,diff --git a/src/collection.cpp b/src/collection.cppindex 123c93c3..34a5be56 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1714,7 +1714,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::string conversation_id,const std::string& override_tags_str,const std::string& voice_query,- bool enable_typos_for_numerical_tokens) const {+ bool enable_typos_for_numerical_tokens,+ bool enable_synonyms) const {std::shared_lock lock(mutex);// setup thread local vars@@ -2304,7 +2305,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,std::unique_ptr<search_args> search_params_guard(search_params);- auto search_op = index->run_search(search_params, name, facet_index_type, enable_typos_for_numerical_tokens);+ auto search_op = index->run_search(search_params, name, facet_index_type,+ enable_typos_for_numerical_tokens, enable_synonyms);// filter_tree_root might be updated in Index::static_filter_query_eval.filter_tree_root_guard.release();diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 7f1a8aa2..e1af018b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1369,6 +1369,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *HIDDEN_HITS = "hidden_hits";const char *ENABLE_OVERRIDES = "enable_overrides";const char *FILTER_CURATED_HITS = "filter_curated_hits";+ const char *ENABLE_SYNONYMS = "enable_synonyms";const char *MAX_CANDIDATES = "max_candidates";@@ -1524,6 +1525,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rebool prioritize_token_position = false;bool pre_segmented_query = false;bool enable_overrides = true;+ bool enable_synonyms = true;size_t filter_curated_hits_option = 2;std::string highlight_fields;bool exhaustive_search = false;@@ -1608,6 +1610,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},{GROUP_MISSING_VALUES, &group_missing_values},{ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},+ {ENABLE_SYNONYMS, &enable_synonyms},};std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {@@ -1822,7 +1825,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconversation_id,override_tags,voice_query,- enable_typos_for_numerical_tokens);+ enable_typos_for_numerical_tokens,+ enable_synonyms);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --git a/src/index.cpp b/src/index.cppindex 1e712213..d3f9dfdd 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2211,7 +2211,8 @@ Option<filter_result_t> Index::do_filtering_with_reference_ids(const std::string}Option<bool> Index::run_search(search_args* search_params, const std::string& collection_name,- facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens) {+ facet_index_type_t facet_index_type, bool enable_typos_for_numerical_tokens,+ bool enable_synonyms) {return search(search_params->field_query_tokens,search_params->search_fields,search_params->match_type,@@ -2252,7 +2253,8 @@ Option<bool> Index::run_search(search_args* search_params, const std::string& cocollection_name,search_params->drop_tokens_mode,facet_index_type,- enable_typos_for_numerical_tokens+ enable_typos_for_numerical_tokens,+ enable_synonyms);}@@ -2740,7 +2742,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consconst std::string& collection_name,const drop_tokens_param_t drop_tokens_mode,facet_index_type_t facet_index_type,- bool enable_typos_for_numerical_tokens) const {+ bool enable_typos_for_numerical_tokens,+ bool enable_synonyms) const {std::shared_lock lock(mutex);auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,@@ -3083,7 +3086,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consfor(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);}- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);++ if(enable_synonyms) {+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);+ }if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 6a0b9707..468c7342 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -1066,4 +1066,87 @@ TEST_F(CollectionSynonymsTest, MultipleSynonymSubstitution) {res = coll2->search("suit man", {"title", "gender"}, "", {},{}, {0}, 10, 1, FREQUENCY, {true}, 0).get();ASSERT_EQ(1, res["hits"].size());+}++TEST_F(CollectionSynonymsTest, EnableSynonymFlag) {+ nlohmann::json schema = R"({+ "name": "coll2",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "gender", "type": "string"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll2 = op.get();++ std::vector<std::vector<std::string>> records = {+ {"Beautiful Blazer", "Male"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["gender"] = records[i][1];++ auto add_op = coll2->add(doc.dump());+ ASSERT_TRUE(add_op.ok());+ }++ nlohmann::json synonym1 = R"({+ "id": "foobar",+ "synonyms": ["blazer", "suit"]+ })"_json;++ nlohmann::json synonym2 = R"({+ "id": "foobar2",+ "synonyms": ["male", "man"]+ })"_json;+++ ASSERT_TRUE(coll2->add_synonym(synonym1).ok());+ ASSERT_TRUE(coll2->add_synonym(synonym2).ok());+ bool enable_synonyms = true;++ auto res = coll2->search("suit man", {"title", "gender"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, enable_synonyms).get();++ ASSERT_EQ(1, res["hits"].size());++ enable_synonyms = false;+ res = coll2->search("suit man", {"title", "gender"}, "", {},+ {}, {2}, 10, 1,FREQUENCY, {true},+ Index::DROP_TOKENS_THRESHOLD, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "",+ 30, 4, "", 40,+ {}, {}, {}, 0,"<mark>",+ "</mark>", {}, 1000,true,+ false, true, "", false,+ 6000*1000, 4, 7, fallback, 4,+ {off}, INT16_MAX, INT16_MAX,2,+ 2, false, "", true,+ 0, max_score, 100, 0, 0,+ HASH, 30000, 2, "",+ {},{}, "right_to_left", true,+ true, false, "", "", "",+ "", false, enable_synonyms).get();++ ASSERT_EQ(0, res["hits"].size());}\ No newline at end of file