Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch 'v0.26-facets' into v27

Kishore Nallane7f9bd30bd81
diff --cc include/collection.h
index dad4df0c,b4e36cbe..353cf9c3
--- a/include/collection.h
+++ b/include/collection.h
@@@ -585,9 -585,7 +585,10 @@@ public
const std::string& override_tags_str = "",
const std::string& voice_query = "",
bool enable_typos_for_numerical_tokens = true,
+ bool enable_synonyms = true,
+ bool synonym_prefix = false,
- uint32_t synonym_num_typos = 0) const;
++ uint32_t synonym_num_typos = 0,
+ bool enable_lazy_filter = true) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --cc include/index.h
index aac65d64,4347cc9f..8e917065
--- a/include/index.h
+++ b/include/index.h
@@@ -691,8 -693,7 +694,10 @@@ public
const drop_tokens_param_t drop_tokens_mode,
facet_index_type_t facet_index_type = DETECT,
bool enable_typos_for_numerical_tokens = true,
+ bool enable_synonyms = true,
- bool synonym_prefix = false, uint32_t synonym_num_typos = 0
++ bool synonym_prefix = false,
++ uint32_t synonym_num_typos = 0,
+ bool enable_lazy_filter = true
) const;
void remove_field(uint32_t seq_id, const nlohmann::json& document, const std::string& field_name,
diff --cc src/collection.cpp
index c8f8e1ff,989bf071..2c41393b
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@@ -1751,10 -1758,7 +1760,10 @@@ Option<nlohmann::json> Collection::sear
const std::string& override_tags_str,
const std::string& voice_query,
bool enable_typos_for_numerical_tokens,
+ bool enable_synonyms,
+ bool synonym_prefix,
- uint32_t synonyms_num_typos) const {
-
++ uint32_t synonyms_num_typos,
+ bool enable_lazy_filter) const {
std::shared_lock lock(mutex);
// setup thread local vars
diff --cc src/collection_manager.cpp
index c6dafa6f,7fa1cc8f..976c14dc
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@@ -1446,10 -1475,8 +1479,11 @@@ Option<bool> CollectionManager::do_sear
const char *VOICE_QUERY = "voice_query";
const char *ENABLE_TYPOS_FOR_NUMERICAL_TOKENS = "enable_typos_for_numerical_tokens";
+ const char *ENABLE_LAZY_FILTER = "enable_lazy_filter";
+ const char *SYNONYM_PREFIX = "synonym_prefix";
+ const char *SYNONYM_NUM_TYPOS = "synonym_num_typos";
+
// enrich params with values from embedded params
for(auto& item: embedded_params.items()) {
if(item.key() == "expires_at") {
@@@ -1646,8 -1669,7 +1681,9 @@@
{PRIORITIZE_NUM_MATCHING_FIELDS, &prioritize_num_matching_fields},
{GROUP_MISSING_VALUES, &group_missing_values},
{ENABLE_TYPOS_FOR_NUMERICAL_TOKENS, &enable_typos_for_numerical_tokens},
+ {ENABLE_SYNONYMS, &enable_synonyms},
+ {SYNONYM_PREFIX, &synonym_prefix},
+ {ENABLE_LAZY_FILTER, &enable_lazy_filter},
};
std::unordered_map<std::string, std::vector<std::string>*> str_list_values = {
@@@ -1863,9 -1885,7 +1899,10 @@@
override_tags,
voice_query,
enable_typos_for_numerical_tokens,
+ enable_synonyms,
+ synonym_prefix,
- synonym_num_typos);
++ synonym_num_typos,
+ enable_lazy_filter);
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
diff --cc src/index.cpp
index 2e34f0cf,479fee6a..5d51487c
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -2254,9 -2253,8 +2254,11 @@@ Option<bool> Index::run_search(search_a
search_params->drop_tokens_mode,
facet_index_type,
enable_typos_for_numerical_tokens,
+ enable_synonyms,
+ synonym_prefix,
- synonym_num_typos);
++ synonym_num_typos,
+ search_params->enable_lazy_filter
+ );
}
void Index::collate_included_ids(const std::vector<token_t>& q_included_tokens,
@@@ -2744,9 -2742,7 +2746,9 @@@ Option<bool> Index::search(std::vector<
const drop_tokens_param_t drop_tokens_mode,
facet_index_type_t facet_index_type,
bool enable_typos_for_numerical_tokens,
+ bool enable_synonyms, bool synonym_prefix,
- uint32_t synonym_num_typos) const {
-
++ uint32_t synonym_num_typos,
+ bool enable_lazy_filter) const {
std::shared_lock lock(mutex);
auto filter_result_iterator = new filter_result_iterator_t(collection_name, this, filter_tree_root,
@@@ -3089,13 -3085,10 +3091,14 @@@
for(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {
q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);
}
- synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
+
+ if(enable_synonyms) {
+ synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms,
+ synonym_prefix, synonym_num_typos);
+ }
- if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {
+ const bool& do_stemming = (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem);
+ if (do_stemming) {
auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();
for(auto& q_include_token: q_include_tokens) {
q_include_token = stemmer->stem(q_include_token);
diff --cc test/collection_synonyms_test.cpp
index 9d9eebac,e927a1f7..9fa0ff50
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@@ -1414,3 -1160,31 +1414,31 @@@ TEST_F(CollectionSynonymsTest, Synonyms
ASSERT_FALSE(synonym_op.ok());
ASSERT_EQ("Invalid offset param.", synonym_op.error());
}
+
+ TEST_F(CollectionSynonymsTest, SynonymWithStemming) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "name", "type": "string", "stem": true}
+ ]
+ })"_json;
+
+ auto coll1 = collectionManager.create_collection(schema).get();
+ std::vector<std::string> records = {"k8s", "kubernetes"};
+
+ for(size_t i = 0; i < records.size(); i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["name"] = records[i];
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ coll1->add_synonym(R"({"id": "syn-1", "synonyms": ["k8s", "kubernetes"]})"_json);
+
+ auto res = coll1->search("k8s", {"name"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ(2, res["found"].get<uint32_t>());
+
+ collectionManager.drop_collection("coll1");
-}
++}