Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Drop tokens should not happen on split tokens.

Kishore Nallandcdb71f87b73
diff --git a/include/index.h b/include/index.h
index b8f7cd0a..6d4ab7d7 100644
--- a/include/index.h
+++ b/include/index.h
@@ -314,6 +314,7 @@ struct search_args {
const size_t max_extra_suffix;
const size_t facet_query_num_typos;
const bool filter_curated_hits;
+ const bool split_join_tokens;
spp::sparse_hash_set<uint64_t> groups_processed;
std::vector<std::vector<art_leaf*>> searched_queries;
@@ -333,7 +334,7 @@ struct search_args {
size_t concurrency, const std::vector<const override_t*>& dynamic_overrides, size_t search_cutoff_ms,
size_t min_len_1typo, size_t min_len_2typo, size_t max_candidates, const std::vector<infix_t>& infixes,
const size_t max_extra_prefix, const size_t max_extra_suffix, const size_t facet_query_num_typos,
- const bool filter_curated_hits) :
+ const bool filter_curated_hits, const bool split_join_tokens) :
field_query_tokens(field_query_tokens),
search_fields(search_fields), filters(filters), facets(facets),
included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),
@@ -346,7 +347,8 @@ struct search_args {
filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms),
min_len_1typo(min_len_1typo), min_len_2typo(min_len_2typo), max_candidates(max_candidates),
infixes(infixes), max_extra_prefix(max_extra_prefix), max_extra_suffix(max_extra_suffix),
- facet_query_num_typos(facet_query_num_typos), filter_curated_hits(filter_curated_hits) {
+ facet_query_num_typos(facet_query_num_typos), filter_curated_hits(filter_curated_hits),
+ split_join_tokens(split_join_tokens) {
const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatory
topster = new Topster(topster_size, group_limit);
@@ -613,7 +615,7 @@ private:
const std::string &field_name,
nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);
- bool common_results_exist(std::vector<art_leaf*>& leaves);
+ bool common_results_exist(std::vector<art_leaf*>& leaves) const;
public:
// for limiting number of results on multiple candidates / query rewrites
@@ -708,7 +710,7 @@ public:
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
- const bool filter_curated_hits) const;
+ const bool filter_curated_hits, bool split_join_tokens) const;
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);
@@ -786,7 +788,7 @@ public:
std::vector<facet_info_t>& facet_infos) const;
void resolve_space_as_typos(std::vector<std::string>& qtokens, const std::string& field_name,
- std::vector<std::vector<std::string>>& resolved_queries);
+ std::vector<std::vector<std::string>>& resolved_queries) const;
size_t num_seq_ids() const;
@@ -845,7 +847,7 @@ public:
const size_t max_extra_suffix, uint32_t* filter_ids, uint32_t filter_ids_length,
const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,
const size_t num_search_fields, const uint32_t* exclude_token_ids, size_t exclude_token_ids_size,
- std::vector<Topster*>& ftopsters, bool is_wildcard_query,
+ std::vector<Topster*>& ftopsters, bool is_wildcard_query, bool split_join_tokens,
std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t>& the_fields, size_t& all_result_ids_len,
uint32_t*& all_result_ids,
diff --git a/src/collection.cpp b/src/collection.cpp
index 438a2bae..59a12633 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1005,19 +1005,6 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
// get synonyms
synonym_reduction(field_query_tokens[i].q_include_tokens, field_query_tokens[i].q_synonyms);
-
- std::vector<std::vector<std::string>> space_resolved_queries;
-
- if(split_join_tokens) {
- index->resolve_space_as_typos(field_query_tokens[i].q_include_tokens, search_field,
- space_resolved_queries);
- }
-
- // only one query is resolved for now, so just use that
- if(!space_resolved_queries.empty()) {
- field_query_tokens[i].q_include_tokens = space_resolved_queries[0];
- synonym_reduction(space_resolved_queries[0], field_query_tokens[i].q_synonyms);
- }
}
}
@@ -1035,7 +1022,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
search_stop_millis,
min_len_1typo, min_len_2typo, max_candidates, infixes,
max_extra_prefix, max_extra_suffix, facet_query_num_typos,
- filter_curated_hits);
+ filter_curated_hits, split_join_tokens);
index->run_search(search_params);
diff --git a/src/index.cpp b/src/index.cpp
index 8365dc73..d11fcba4 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1654,7 +1654,8 @@ void Index::run_search(search_args* search_params) {
search_params->max_extra_prefix,
search_params->max_extra_suffix,
search_params->facet_query_num_typos,
- search_params->filter_curated_hits);
+ search_params->filter_curated_hits,
+ search_params->split_join_tokens);
}
void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,
@@ -2101,7 +2102,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
size_t concurrency, size_t search_cutoff_ms, size_t min_len_1typo, size_t min_len_2typo,
size_t max_candidates, const std::vector<infix_t>& infixes, const size_t max_extra_prefix,
const size_t max_extra_suffix, const size_t facet_query_num_typos,
- const bool filter_curated_hits) const {
+ const bool filter_curated_hits, const bool split_join_tokens) const {
search_begin = std::chrono::high_resolution_clock::now();
search_stop_ms = search_cutoff_ms;
@@ -2221,7 +2222,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
concurrency, min_len_1typo, min_len_2typo, max_candidates, infixes, max_extra_prefix,
max_extra_suffix,
filter_ids, filter_ids_length, curated_ids, curated_ids_sorted, num_search_fields,
- exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, field_query_tokens,
+ exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, false, field_query_tokens,
the_fields, all_result_ids_len, all_result_ids, topster_ids);
if(exhaustive_search || all_result_ids_len < typo_tokens_threshold || all_result_ids_len < drop_tokens_threshold) {
@@ -2232,8 +2233,8 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens, const std::v
concurrency, min_len_1typo, min_len_2typo, max_candidates, infixes, max_extra_prefix,
max_extra_suffix,
filter_ids, filter_ids_length, curated_ids, curated_ids_sorted, num_search_fields,
- exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, field_query_tokens,
- the_fields, all_result_ids_len, all_result_ids, topster_ids);
+ exclude_token_ids, exclude_token_ids_size, ftopsters, is_wildcard_query, split_join_tokens,
+ field_query_tokens, the_fields, all_result_ids_len, all_result_ids, topster_ids);
}
//auto begin0 = std::chrono::high_resolution_clock::now();
@@ -2514,7 +2515,7 @@ void Index::search_fields(const std::vector<filter>& filters,
const std::set<uint32_t>& curated_ids, const std::vector<uint32_t>& curated_ids_sorted,
const size_t num_search_fields, const uint32_t* exclude_token_ids,
size_t exclude_token_ids_size, std::vector<Topster*>& ftopsters, bool is_wildcard_query,
- std::vector<query_tokens_t>& field_query_tokens,
+ const bool split_join_tokens, std::vector<query_tokens_t>& field_query_tokens,
const std::vector<search_field_t>& the_fields, size_t& all_result_ids_len,
uint32_t*& all_result_ids,
spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const {
@@ -2552,7 +2553,6 @@ void Index::search_fields(const std::vector<filter>& filters,
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.empty() || actual_filter_ids_length > 0) {
const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
- const std::string& field_name = the_fields[i].name;
auto field_it = search_schema.find(field_name);
if(field_it == search_schema.end()) {
@@ -2588,6 +2588,36 @@ void Index::search_fields(const std::vector<filter>& filters,
max_extra_prefix, max_extra_suffix, field_infix, field_id, field_name, query_tokens,
actual_topster, filter_ids, filter_ids_length,
curated_ids_sorted, field_num_results, all_result_ids, all_result_ids_len, groups_processed);
+
+ if(field_num_results == 0 && split_join_tokens) {
+ std::vector<std::vector<std::string>> space_resolved_queries;
+ resolve_space_as_typos(field_query_tokens[i].q_include_tokens, field_name,
+ space_resolved_queries);
+
+ // only one query is resolved for now, so just use that
+ if(!space_resolved_queries.empty()) {
+ q_include_pos_tokens.clear();
+ const auto& resolved_query = space_resolved_queries[0];
+
+ for(size_t j=0; j < resolved_query.size(); j++) {
+ bool is_prefix = (j == resolved_query.size()-1);
+ q_include_pos_tokens.emplace_back(j, space_resolved_queries[0][j], is_prefix);
+ }
+
+ query_tokens = q_include_pos_tokens;
+ search_tokens = q_include_pos_tokens;
+
+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size,
+ num_tokens_dropped, field_it->second, field_name,
+ actual_filter_ids, actual_filter_ids_length, curated_ids_sorted, sort_fields_std,
+ field_num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
+ field_num_results, group_limit, group_by_fields, prioritize_exact_match, concurrency,
+ query_hashes, token_order, field_prefix,
+ 0, 0, exhaustive_search,
+ min_len_1typo, min_len_2typo, max_candidates);
+ }
+ }
+
} else if(actual_filter_ids_length != 0) {
// indicates phrase match query
curate_filtered_ids(filters, curated_ids, exclude_token_ids,
@@ -4386,9 +4416,7 @@ size_t Index::num_seq_ids() const {
}
void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const string& field_name,
- std::vector<std::vector<std::string>>& resolved_queries) {
-
- std::shared_lock lock(mutex);
+ std::vector<std::vector<std::string>>& resolved_queries) const {
auto tree_it = search_index.find(field_name);
@@ -4523,7 +4551,7 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const stri
}
}
-bool Index::common_results_exist(std::vector<art_leaf*>& leaves) {
+bool Index::common_results_exist(std::vector<art_leaf*>& leaves) const {
std::vector<uint32_t> result_ids;
std::vector<void*> leaf_vals;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 808e6e01..9faa484d 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1924,7 +1924,6 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {
nlohmann::json doc;
doc["title"] = "The nonstick pressure cooker is a great invention.";
-
ASSERT_TRUE(coll1->add(doc.dump()).ok());
auto results = coll1->search("non stick", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();
@@ -1968,6 +1967,16 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {
"<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000*1000, 4, 7, false).get();
ASSERT_EQ(0, results["hits"].size());
+ // drop tokens should not happen on tokens split
+ doc["title"] = "Pressure Copper vessel";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ results = coll1->search("pressurecopper", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("Pressure Copper vessel", results["hits"][0]["document"]["title"].get<std::string>());
+
collectionManager.drop_collection("coll1");
}