Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch 'v0.26-facets' into v27
Kishore Nallane7f9bd30bd81
diff --cc include/collection.hindex dad4df0c,b4e36cbe..353cf9c3--- a/include/collection.h+++ b/include/collection.h@@@ -585,9 -585,7 +585,10 @@@ publicconst std::string& override_tags_str = "",const std::string& voice_query = "",bool enable_typos_for_numerical_tokens = true,+ bool enable_synonyms = true,+ bool synonym_prefix = false,- uint32_t synonym_num_typos = 0) const;++ uint32_t synonym_num_typos = 0,+ bool enable_lazy_filter = true) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --cc include/index.hindex aac65d64,4347cc9f..8e917065--- a/include/index.h+++ b/include/index.h@@@ -691,8 -693,7 +694,10 @@@ publicconst drop_tokens_param_t drop_tokens_mode,facet_index_type_t facet_index_type = DETECT,bool enable_typos_for_numerical_tokens = true,+ bool enable_synonyms = true,- bool synonym_prefix = false, uint32_t synonym_num_typos = 0++ bool synonym_prefix = false,++ uint32_t synonym_num_typos = 0,+ bool enable_lazy_filter = true) const;void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,diff --cc src/collection.cppindex c8f8e1ff,989bf071..2c41393b--- a/src/collection.cpp+++ b/src/collection.cpp@@@ -1751,10 -1758,7 +1760,10 @@@ Option<nlohmann::json> Collection::searconst std::string& override_tags_str,const std::string& voice_query,bool enable_typos_for_numerical_tokens,+ bool enable_synonyms,+ bool synonym_prefix,- uint32_t synonyms_num_typos) const {-++ uint32_t synonyms_num_typos,+ bool enable_lazy_filter) const {std::shared_lock lock(mutex);// setup thread local varsdiff --cc src/collection_manager.cppindex c6dafa6f,7fa1cc8f..976c14dc--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@@ -1446,10 -1475,8 +1479,11 @@@ Option<bool> CollectionManager::do_searconst char *VOICE_QUERY = "voice_query";const char *ENABLE_TYPOS_FOR_NUMERICAL_TOKENS = "enable_typos_for_numerical_tokens";+ const char *ENABLE_LAZY_FILTER = "enable_lazy_filter";+ const char *SYNONYM_PREFIX = "synonym_prefix";+ const char *SYNONYM_NUM_TYPOS = "synonym_num_typos";+// enrich params with values from embedded paramsfor(auto& item: embedded_params.items()) {if(item.key() == "expires_at") {@@@ -1646,8 -1669,7 +1681,9 @@@{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},{GROUP_MISSING_VALUES, &group_missing_values},{ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},+ {ENABLE_SYNONYMS, &enable_synonyms},+ {SYNONYM_PREFIX, &synonym_prefix},+ {ENABLE_LAZY_FILTER, &enable_lazy_filter},};std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {@@@ -1863,9 -1885,7 +1899,10 @@@override_tags,voice_query,enable_typos_for_numerical_tokens,+ enable_synonyms,+ synonym_prefix,- synonym_num_typos);++ synonym_num_typos,+ enable_lazy_filter);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();diff --cc src/index.cppindex 2e34f0cf,479fee6a..5d51487c--- a/src/index.cpp+++ b/src/index.cpp@@@ -2254,9 -2253,8 +2254,11 @@@ Option<bool> Index::run_search(search_asearch_params->drop_tokens_mode,facet_index_type,enable_typos_for_numerical_tokens,+ enable_synonyms,+ synonym_prefix,- synonym_num_typos);++ synonym_num_typos,+ search_params->enable_lazy_filter+ );}void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,@@@ -2744,9 -2742,7 +2746,9 @@@ Option<bool> Index::search(std::vector<const drop_tokens_param_t drop_tokens_mode,facet_index_type_t facet_index_type,bool enable_typos_for_numerical_tokens,+ bool enable_synonyms, bool synonym_prefix,- uint32_t synonym_num_typos) const {-++ uint32_t synonym_num_typos,+ bool enable_lazy_filter) const {std::shared_lock lock(mutex);auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,@@@ -3089,13 -3085,10 +3091,14 @@@for(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);}- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);++ if(enable_synonyms) {+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms,+ synonym_prefix, synonym_num_typos);+ }- if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {+ const bool& do_stemming = (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem);+ if (do_stemming) {auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();for(auto& q_include_token: q_include_tokens) {q_include_token = stemmer->stem(q_include_token);diff --cc test/collection_synonyms_test.cppindex 9d9eebac,e927a1f7..9fa0ff50--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@@ -1414,3 -1160,31 +1414,31 @@@ TEST_F(CollectionSynonymsTest, SynonymsASSERT_FALSE(synonym_op.ok());ASSERT_EQ("Invalid offset param.", synonym_op.error());}++ TEST_F(CollectionSynonymsTest, SynonymWithStemming) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "name", "type": "string", "stem": true}+ ]+ })"_json;++ auto coll1 = collectionManager.create_collection(schema).get();+ std::vector<std::string> records = {"k8s", "kubernetes"};++ for(size_t i = 0; i < records.size(); i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["name"] = records[i];+ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ coll1->add_synonym(R"({"id": "syn-1", "synonyms": ["k8s", "kubernetes"]})"_json);++ auto res = coll1->search("k8s", {"name"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(2, res["hits"].size());+ ASSERT_EQ(2, res["found"].get<uint32_t>());++ collectionManager.drop_collection("coll1");-}++}