Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Option to curtail expensive queries.
Kishore Nallan6210e823907c
diff --git a/include/collection.h b/include/collection.hindex 2b3baba4..e1942d37 100644--- a/include/collection.h+++ b/include/collection.h@@ -386,7 +386,8 @@ public:bool pre_segmented_query=false,bool enable_overrides=true,const std::string& highlight_fields="",- const bool exhaustive_search = false) const;+ const bool exhaustive_search = false,+ size_t search_stop_millis = 200) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/index.h b/include/index.hindex 6af2c749..8d8171b2 100644--- a/include/index.h+++ b/include/index.h@@ -260,6 +260,7 @@ struct search_args {bool exhaustive_search;size_t concurrency;const std::vector<const override_t*>& filter_overrides;+ size_t search_cutoff_ms;spp::sparse_hash_set<uint64_t> groups_processed;std::vector<std::vector<art_leaf*>> searched_queries;@@ -281,7 +282,8 @@ struct search_args {bool prioritize_exact_match,bool exhaustive_search,size_t concurrency,- const std::vector<const override_t*>& dynamic_overrides):+ const std::vector<const override_t*>& dynamic_overrides,+ size_t search_cutoff_ms):field_query_tokens(field_query_tokens),search_fields(search_fields), filters(filters), facets(facets),included_ids(included_ids), excluded_ids(excluded_ids), sort_fields_std(sort_fields_std),@@ -291,7 +293,7 @@ struct search_args {group_by_fields(group_by_fields), group_limit(group_limit), default_sorting_field(default_sorting_field),prioritize_exact_match(prioritize_exact_match), all_result_ids_len(0),exhaustive_search(exhaustive_search), concurrency(concurrency),- filter_overrides(dynamic_overrides) {+ filter_overrides(dynamic_overrides), search_cutoff_ms(search_cutoff_ms) {const size_t topster_size = std::max((size_t)1, max_hits); // needs to be atleast 1 since scoring is mandatorytopster = new Topster(topster_size, group_limit);@@ -650,7 +652,8 @@ public:const std::string& default_sorting_field,bool prioritize_exact_match,bool exhaustive_search,- size_t concurrency) const;+ size_t concurrency,+ size_t search_cutoff_ms) const;Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document, const bool is_update);diff --git a/include/posting.h b/include/posting.hindex 682135b3..85ab60e9 100644--- a/include/posting.h+++ b/include/posting.h@@ -22,7 +22,7 @@ struct compact_posting_list_t {static compact_posting_list_t* create(uint32_t num_ids, const uint32_t* ids, const uint32_t* offset_index,uint32_t num_offsets, uint32_t* offsets);- posting_list_t* to_full_posting_list();+ posting_list_t* to_full_posting_list() const;bool contains(uint32_t id);diff --git a/include/thread_local_vars.h b/include/thread_local_vars.hindex 37845ac5..0b5f024d 100644--- a/include/thread_local_vars.h+++ b/include/thread_local_vars.h@@ -1 +1,6 @@-extern thread_local int64_t write_log_index;\ No newline at end of file+#include <chrono>++extern thread_local int64_t write_log_index;+extern thread_local std::chrono::high_resolution_clock::time_point begin;+extern thread_local int64_t search_stop_ms;+extern thread_local bool search_cutoff;\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex c92da6ff..020b02d8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -493,7 +493,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sbool pre_segmented_query,bool enable_overrides,const std::string& highlight_fields,- const bool exhaustive_search) const {+ const bool exhaustive_search,+ const size_t search_stop_millis) const {std::shared_lock lock(mutex);@@ -916,7 +917,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sper_page, page, token_order, prefixes,drop_tokens_threshold, typo_tokens_threshold,group_by_fields, group_limit, default_sorting_field, prioritize_exact_match,- exhaustive_search, 4, filter_overrides);+ exhaustive_search, 4, filter_overrides,+ search_stop_millis);index->run_search(search_params);@@ -1251,6 +1253,8 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s// free search paramsdelete search_params;+ result["search_cutoff"] = search_cutoff;+result["request_params"] = nlohmann::json::object();;result["request_params"]["collection_name"] = name;result["request_params"]["per_page"] = per_page;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 2935d2bf..0034c71b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -535,6 +535,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *PRE_SEGMENTED_QUERY = "pre_segmented_query";const char *EXHAUSTIVE_SEARCH = "exhaustive_search";+ const char *SEARCH_CUTOFF_MS = "search_cutoff_ms";if(req_params.count(NUM_TYPOS) == 0) {req_params[NUM_TYPOS] = "2";@@ -637,6 +638,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rereq_params[EXHAUSTIVE_SEARCH] = "false";}+ if(req_params.count(SEARCH_CUTOFF_MS) == 0) {+ req_params[SEARCH_CUTOFF_MS] = "2000";+ }+std::vector<std::string> query_by_weights_str;std::vector<size_t> query_by_weights;@@ -706,6 +711,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rereturn Option<bool>(400,"Parameter `" + std::string(GROUP_LIMIT) + "` must be an unsigned integer.");}+ if(!StringUtils::is_uint32_t(req_params[SEARCH_CUTOFF_MS])) {+ return Option<bool>(400,"Parameter `" + std::string(SEARCH_CUTOFF_MS) + "` must be an unsigned integer.");+ }+bool prioritize_exact_match = (req_params[PRIORITIZE_EXACT_MATCH] == "true");bool pre_segmented_query = (req_params[PRE_SEGMENTED_QUERY] == "true");bool exhaustive_search = (req_params[EXHAUSTIVE_SEARCH] == "true");@@ -813,7 +822,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& repre_segmented_query,enable_overrides,req_params[HIGHLIGHT_FIELDS],- exhaustive_search+ exhaustive_search,+ static_cast<size_t>(std::stol(req_params[SEARCH_CUTOFF_MS])));uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/index.cpp b/src/index.cppindex 1c61730f..12e871e6 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -17,8 +17,21 @@#include <s2/s2loop.h>#include <s2/s2builder.h>#include <posting.h>+#include <thread_local_vars.h>#include "logger.h"+#define RETURN_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\+ std::chrono::high_resolution_clock::now() - begin).count() > search_stop_ms) { \+ search_cutoff = true; \+ return ;\+ }++#define BREAK_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\+ std::chrono::high_resolution_clock::now() - begin).count() > search_stop_ms) { \+ search_cutoff = true; \+ break;\+ }+spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::geo_sentinel_value;@@ -1113,6 +1126,8 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,size_t combination_limit = exhaustive_search ? Index::COMBINATION_MAX_LIMIT : Index::COMBINATION_MIN_LIMIT;for(long long n=0; n<N && n<combination_limit; ++n) {+ RETURN_CIRCUIT_BREAKER+// every element in `query_suggestion` contains a token and its associated hitsstd::vector<art_leaf*> query_suggestion(token_candidates_vec.size());@@ -1232,6 +1247,8 @@ void Index::do_filtering(uint32_t*& filter_ids, uint32_t& filter_ids_length,for(size_t i = 0; i < filters.size(); i++) {const filter & a_filter = filters[i];+ RETURN_CIRCUIT_BREAKER+if(a_filter.field_name == "id") {// we handle `ids` separatelystd::vector<uint32> result_ids;@@ -1616,7 +1633,8 @@ void Index::run_search(search_args* search_params) {search_params->default_sorting_field,search_params->prioritize_exact_match,search_params->exhaustive_search,- search_params->concurrency);+ search_params->concurrency,+ search_params->search_cutoff_ms);}void Index::collate_included_ids(const std::vector<std::string>& q_included_tokens,@@ -2002,9 +2020,11 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,const std::string& default_sorting_field,bool prioritize_exact_match,const bool exhaustive_search,- const size_t concurrency) const {+ const size_t concurrency,+ const size_t search_cutoff_ms) const {- //auto begin = std::chrono::high_resolution_clock::now();+ begin = std::chrono::high_resolution_clock::now();+ search_stop_ms = search_cutoff_ms;// process the filters@@ -2520,12 +2540,19 @@ void Index::search_wildcard(const std::vector<std::string>& qtokens, const std::uint32_t token_bits = 255;std::vector<posting_list_t::iterator_t> plists;+ bool check_for_circuit_break = (filter_ids_length > 1000000);+for(size_t i = 0; i < filter_ids_length; i++) {const uint32_t seq_id = filter_ids[i];score_results(sort_fields_std, (uint16_t) searched_queries.size(), field_id, false, 0, topster, {},groups_processed, seq_id, sort_order, field_values,geopoint_indices, group_limit, group_by_fields, token_bits,false, false, plists);++ if(check_for_circuit_break && i % (2^17) == 0) {+ // check only once every 2^17 docs to reduce overhead+ BREAK_CIRCUIT_BREAKER+ }}collate_included_ids(qtokens, field, field_id, included_ids_map, curated_topster, searched_queries);@@ -2637,6 +2664,8 @@ void Index::search_field(const uint8_t & field_id,const size_t num_fuzzy_candidates = exhaustive_search ? 10000 : 4;while(n < N && n < combination_limit) {+ RETURN_CIRCUIT_BREAKER+// Outerloop generates combinations of [cost to max_cost] for each token// For e.g. for a 3-token query: [0, 0, 0], [0, 0, 1], [0, 1, 1] etc.std::vector<uint32_t> costs(token_to_costs.size());diff --git a/src/posting.cpp b/src/posting.cppindex 6b2ef148..33158b56 100644--- a/src/posting.cpp+++ b/src/posting.cpp@@ -155,7 +155,7 @@ compact_posting_list_t* compact_posting_list_t::create(uint32_t num_ids, const ureturn pl;}-posting_list_t* compact_posting_list_t::to_full_posting_list() {+posting_list_t* compact_posting_list_t::to_full_posting_list() const {posting_list_t* pl = new posting_list_t(256);size_t i = 0;diff --git a/src/thread_local_vars.cpp b/src/thread_local_vars.cppindex 79ba502b..816a898e 100644--- a/src/thread_local_vars.cpp+++ b/src/thread_local_vars.cpp@@ -2,3 +2,6 @@#include "thread_local_vars.h"thread_local int64_t write_log_index = 0;+thread_local std::chrono::high_resolution_clock::time_point begin;+thread_local int64_t search_stop_ms;+thread_local bool search_cutoff = false;diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex efe69fdd..b8cb8ace 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -526,5 +526,47 @@ TEST_F(CollectionSynonymsTest, SynonymJsonSerialization) {ASSERT_STREQ("ipod", obj["synonyms"][0].get<std::string>().c_str());ASSERT_STREQ("i pod", obj["synonyms"][1].get<std::string>().c_str());ASSERT_STREQ("pod", obj["synonyms"][2].get<std::string>().c_str());+}++TEST_F(CollectionSynonymsTest, SynonymSingleTokenExactMatch) {+ Collection *coll1;+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("description", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"Smashed Lemon", "Description 1", "100"},+ {"Lulu Guinness", "Description 2", "100"},+ {"Lululemon", "Description 3", "100"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["description"] = records[i][1];+ doc["points"] = std::stoi(records[i][2]);++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ synonym_t synonym1{"syn-1", {"lulu", "lemon"}, {{"lululemon"}}};+ coll1->add_synonym(synonym1);++ auto res = coll1->search("lulu lemon", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 1).get();++ ASSERT_EQ(2, res["hits"].size());+ ASSERT_EQ(2, res["found"].get<uint32_t>());++ ASSERT_STREQ("2", res["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");}\ No newline at end of file