Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add `filter-by-max-ops` option. (#1619)
Harpreet Sangara27645f6bb71
diff --git a/include/collection_manager.h b/include/collection_manager.hindex a88e3f7b..cc5be3f0 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -105,6 +105,8 @@ public:static constexpr const char* PRESET_PREFIX = "$PS";static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";+ uint16_t filter_by_max_ops;+static CollectionManager & get_instance() {static CollectionManager instance;return instance;@@ -145,10 +147,12 @@ public:// PUBLICLY EXPOSED APIvoid init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,- const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer);+ const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer,+ const uint16_t& filter_by_max_operations = Config::FILTER_BY_DEFAULT_OPERATIONS);// only for tests!- void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);+ void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit,+ const uint16_t& filter_by_max_operations = Config::FILTER_BY_DEFAULT_OPERATIONS);Option<bool> load(const size_t collection_batch_size, const size_t document_batch_size);diff --git a/include/tsconfig.h b/include/tsconfig.hindex b6147caa..1e55fbe6 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -76,6 +76,8 @@ private:bool enable_search_logging;+ uint16_t filter_by_max_ops;+protected:Config() {@@ -107,6 +109,8 @@ protected:this->db_compaction_interval = 0; // in seconds, disabledthis->enable_search_logging = false;++ this->filter_by_max_ops = FILTER_BY_DEFAULT_OPERATIONS;}Config(Config const&) {@@ -115,6 +119,8 @@ protected:public:+ static constexpr uint16_t FILTER_BY_DEFAULT_OPERATIONS = 100;+static Config & get_instance() {static Config instance;return instance;@@ -364,6 +370,10 @@ public:return skip_writes;}+ uint16_t get_filter_by_max_ops() const {+ return filter_by_max_ops;+ }+// loadersstd::string get_env(const char *name) {@@ -496,6 +506,10 @@ public:this->memory_used_max_percentage = std::stoi(get_env("TYPESENSE_MEMORY_USED_MAX_PERCENTAGE"));}+ if(!get_env("TYPESENSE_FILTER_BY_MAX_OPS").empty()) {+ this->filter_by_max_ops = std::stoi(get_env("TYPESENSE_FILTER_BY_MAX_OPS"));+ }+this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR"));}@@ -689,6 +703,10 @@ public:this->reset_peers_on_error = (reset_peers_on_error_str == "true");}+ if(reader.Exists("server", "filter-by-max-ops")) {+ this->filter_by_max_ops = (uint16_t) reader.GetInteger("server", "filter-by-max-ops", FILTER_BY_DEFAULT_OPERATIONS);+ }+}void load_config_cmd_args(cmdline::parser & options) {@@ -854,6 +872,10 @@ public:if(options.exist("enable-search-logging")) {this->enable_search_logging = options.get<bool>("enable-search-logging");}++ if(options.exist("filter-by-max-ops")) {+ this->filter_by_max_ops = options.get<uint16_t>("filter-by-max-ops");+ }}void set_cors_domains(std::string& cors_domains_value) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex e1af018b..e52700ee 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -210,7 +210,8 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,const float max_memory_ratio,const std::string & auth_key,std::atomic<bool>& quit,- BatchedIndexer* batch_indexer) {+ BatchedIndexer* batch_indexer,+ const uint16_t& filter_by_max_operations) {std::unique_lock lock(mutex);this->store = store;@@ -219,13 +220,15 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,this->max_memory_ratio = max_memory_ratio;this->quit = &quit;this->batch_indexer = batch_indexer;+ this->filter_by_max_ops = filter_by_max_operations;}// used only in tests!void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,- std::atomic<bool>& quit) {+ std::atomic<bool>& quit,+ const uint16_t& filter_by_max_operations) {ThreadPool* thread_pool = new ThreadPool(8);- init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr);+ init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr, filter_by_max_operations);}Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {diff --git a/src/filter.cpp b/src/filter.cppindex a8a652a6..65abfcf7 100644--- a/src/filter.cpp+++ b/src/filter.cpp@@ -759,8 +759,9 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,return toPostfix_op;}- if (postfix.size() > 100) {- return Option<bool>(400, "`filter_by` has too many operations.");+ auto const& max_ops = CollectionManager::get_instance().filter_by_max_ops;+ if (postfix.size() > max_ops) {+ return Option<bool>(400, "`filter_by` has too many operations. Maximum allowed: " + std::to_string(max_ops));}Option<bool> toParseTree_op = toParseTree(postfix,diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 5f970d5c..2cf22675 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -112,6 +112,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);options.add<uint32_t>("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800);+ options.add<uint16_t>("filter-by-max-ops", '\0', "Maximum number of operations permitted in filtery_by.", false, Config::FILTER_BY_DEFAULT_OPERATIONS);// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");@@ -436,7 +437,7 @@ int run_server(const Config & config, const std::string & version, void (*masterCollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),- config.get_api_key(), quit_raft_service, batch_indexer);+ config.get_api_key(), quit_raft_service, batch_indexer, config.get_filter_by_max_ops());StopwordsManager& stopwordsManager = StopwordsManager::get_instance();stopwordsManager.init(&store);diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 13e1bf3d..d452caf2 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -8,6 +8,7 @@class CollectionFilteringTest : public ::testing::Test {protected:+ std::string state_dir_path = "/tmp/typesense_test/collection_filtering";Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();std::atomic<bool> quit = false;@@ -16,7 +17,6 @@ protected:std::vector<sort_by> sort_fields;void setupCollection() {- std::string state_dir_path = "/tmp/typesense_test/collection_filtering";LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());@@ -2204,7 +2204,31 @@ TEST_F(CollectionFilteringTest, ComplexFilterQuery) {search_op = coll->search("Jeremy", {"name"}, extreme_filter,{}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false});ASSERT_FALSE(search_op.ok());- ASSERT_EQ("`filter_by` has too many operations.", search_op.error());+ ASSERT_EQ("`filter_by` has too many operations. Maximum allowed: 100", search_op.error());++ collectionManager.dispose();+ delete store;++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key", quit, 109); // Re-initialize with 109 filter operations allowed.+ auto load_op = collectionManager.load(8, 1000);++ if(!load_op.ok()) {+ LOG(ERROR) << load_op.error();+ }+ ASSERT_TRUE(load_op.ok());++ coll = collectionManager.get_collection_unsafe("ComplexFilterQueryCollection");+ search_op = coll->search("Jeremy", {"name"}, extreme_filter,+ {}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false});+ ASSERT_TRUE(search_op.ok());+ ASSERT_EQ(1, search_op.get()["hits"].size());++ extreme_filter += "|| (years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5)))";+ search_op = coll->search("Jeremy", {"name"}, extreme_filter,+ {}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false});+ ASSERT_FALSE(search_op.ok());+ ASSERT_EQ("`filter_by` has too many operations. Maximum allowed: 109", search_op.error());collectionManager.drop_collection("ComplexFilterQueryCollection");}