Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add `filter-by-max-ops` option. (#1619)

Harpreet Sangara27645f6bb71
diff --git a/include/collection_manager.h b/include/collection_manager.h
index a88e3f7b..cc5be3f0 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -105,6 +105,8 @@ public:
static constexpr const char* PRESET_PREFIX = "$PS";
static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";
+ uint16_t filter_by_max_ops;
+
static CollectionManager & get_instance() {
static CollectionManager instance;
return instance;
@@ -145,10 +147,12 @@ public:
// PUBLICLY EXPOSED API
void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,
- const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer);
+ const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer,
+ const uint16_t& filter_by_max_operations = Config::FILTER_BY_DEFAULT_OPERATIONS);
// only for tests!
- void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);
+ void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit,
+ const uint16_t& filter_by_max_operations = Config::FILTER_BY_DEFAULT_OPERATIONS);
Option<bool> load(const size_t collection_batch_size, const size_t document_batch_size);
diff --git a/include/tsconfig.h b/include/tsconfig.h
index b6147caa..1e55fbe6 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -76,6 +76,8 @@ private:
bool enable_search_logging;
+ uint16_t filter_by_max_ops;
+
protected:
Config() {
@@ -107,6 +109,8 @@ protected:
this->db_compaction_interval = 0; // in seconds, disabled
this->enable_search_logging = false;
+
+ this->filter_by_max_ops = FILTER_BY_DEFAULT_OPERATIONS;
}
Config(Config const&) {
@@ -115,6 +119,8 @@ protected:
public:
+ static constexpr uint16_t FILTER_BY_DEFAULT_OPERATIONS = 100;
+
static Config & get_instance() {
static Config instance;
return instance;
@@ -364,6 +370,10 @@ public:
return skip_writes;
}
+ uint16_t get_filter_by_max_ops() const {
+ return filter_by_max_ops;
+ }
+
// loaders
std::string get_env(const char *name) {
@@ -496,6 +506,10 @@ public:
this->memory_used_max_percentage = std::stoi(get_env("TYPESENSE_MEMORY_USED_MAX_PERCENTAGE"));
}
+ if(!get_env("TYPESENSE_FILTER_BY_MAX_OPS").empty()) {
+ this->filter_by_max_ops = std::stoi(get_env("TYPESENSE_FILTER_BY_MAX_OPS"));
+ }
+
this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));
this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR"));
}
@@ -689,6 +703,10 @@ public:
this->reset_peers_on_error = (reset_peers_on_error_str == "true");
}
+ if(reader.Exists("server", "filter-by-max-ops")) {
+ this->filter_by_max_ops = (uint16_t) reader.GetInteger("server", "filter-by-max-ops", FILTER_BY_DEFAULT_OPERATIONS);
+ }
+
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -854,6 +872,10 @@ public:
if(options.exist("enable-search-logging")) {
this->enable_search_logging = options.get<bool>("enable-search-logging");
}
+
+ if(options.exist("filter-by-max-ops")) {
+ this->filter_by_max_ops = options.get<uint16_t>("filter-by-max-ops");
+ }
}
void set_cors_domains(std::string& cors_domains_value) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index e1af018b..e52700ee 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -210,7 +210,8 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,
const float max_memory_ratio,
const std::string & auth_key,
std::atomic<bool>& quit,
- BatchedIndexer* batch_indexer) {
+ BatchedIndexer* batch_indexer,
+ const uint16_t& filter_by_max_operations) {
std::unique_lock lock(mutex);
this->store = store;
@@ -219,13 +220,15 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,
this->max_memory_ratio = max_memory_ratio;
this->quit = &quit;
this->batch_indexer = batch_indexer;
+ this->filter_by_max_ops = filter_by_max_operations;
}
// used only in tests!
void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,
- std::atomic<bool>& quit) {
+ std::atomic<bool>& quit,
+ const uint16_t& filter_by_max_operations) {
ThreadPool* thread_pool = new ThreadPool(8);
- init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr);
+ init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr, filter_by_max_operations);
}
Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {
diff --git a/src/filter.cpp b/src/filter.cpp
index a8a652a6..65abfcf7 100644
--- a/src/filter.cpp
+++ b/src/filter.cpp
@@ -759,8 +759,9 @@ Option<bool> filter::parse_filter_query(const std::string& filter_query,
return toPostfix_op;
}
- if (postfix.size() > 100) {
- return Option<bool>(400, "`filter_by` has too many operations.");
+ auto const& max_ops = CollectionManager::get_instance().filter_by_max_ops;
+ if (postfix.size() > max_ops) {
+ return Option<bool>(400, "`filter_by` has too many operations. Maximum allowed: " + std::to_string(max_ops));
}
Option<bool> toParseTree_op = toParseTree(postfix,
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 5f970d5c..2cf22675 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -112,6 +112,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);
options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);
options.add<uint32_t>("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800);
+ options.add<uint16_t>("filter-by-max-ops", '\0', "Maximum number of operations permitted in filtery_by.", false, Config::FILTER_BY_DEFAULT_OPERATIONS);
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
@@ -436,7 +437,7 @@ int run_server(const Config & config, const std::string & version, void (*master
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
- config.get_api_key(), quit_raft_service, batch_indexer);
+ config.get_api_key(), quit_raft_service, batch_indexer, config.get_filter_by_max_ops());
StopwordsManager& stopwordsManager = StopwordsManager::get_instance();
stopwordsManager.init(&store);
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 13e1bf3d..d452caf2 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -8,6 +8,7 @@
class CollectionFilteringTest : public ::testing::Test {
protected:
+ std::string state_dir_path = "/tmp/typesense_test/collection_filtering";
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
@@ -16,7 +17,6 @@ protected:
std::vector<sort_by> sort_fields;
void setupCollection() {
- std::string state_dir_path = "/tmp/typesense_test/collection_filtering";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
@@ -2204,7 +2204,31 @@ TEST_F(CollectionFilteringTest, ComplexFilterQuery) {
search_op = coll->search("Jeremy", {"name"}, extreme_filter,
{}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false});
ASSERT_FALSE(search_op.ok());
- ASSERT_EQ("`filter_by` has too many operations.", search_op.error());
+ ASSERT_EQ("`filter_by` has too many operations. Maximum allowed: 100", search_op.error());
+
+ collectionManager.dispose();
+ delete store;
+
+ store = new Store(state_dir_path);
+ collectionManager.init(store, 1.0, "auth_key", quit, 109); // Re-initialize with 109 filter operations allowed.
+ auto load_op = collectionManager.load(8, 1000);
+
+ if(!load_op.ok()) {
+ LOG(ERROR) << load_op.error();
+ }
+ ASSERT_TRUE(load_op.ok());
+
+ coll = collectionManager.get_collection_unsafe("ComplexFilterQueryCollection");
+ search_op = coll->search("Jeremy", {"name"}, extreme_filter,
+ {}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false});
+ ASSERT_TRUE(search_op.ok());
+ ASSERT_EQ(1, search_op.get()["hits"].size());
+
+ extreme_filter += "|| (years:>2000 && ((age:<30 && rating:>5) || (age:>50 && rating:<5)))";
+ search_op = coll->search("Jeremy", {"name"}, extreme_filter,
+ {}, sort_fields_desc, {0}, 10, 1, FREQUENCY, {false});
+ ASSERT_FALSE(search_op.ok());
+ ASSERT_EQ("`filter_by` has too many operations. Maximum allowed: 109", search_op.error());
collectionManager.drop_collection("ComplexFilterQueryCollection");
}