Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow a threshold for memory limit to be specified.

kishorenc22ce33bfbe25
diff --git a/include/collection.h b/include/collection.h
index 5c356bd9..92355bc7 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -142,6 +142,8 @@ private:
size_t num_indices;
+ const float max_memory_ratio;
+
std::string get_doc_id_key(const std::string & doc_id);
std::string get_seq_id_key(uint32_t seq_id);
@@ -177,7 +179,7 @@ public:
Collection(const std::string name, const uint32_t collection_id, const uint64_t created_at,
const uint32_t next_seq_id, Store *store, const std::vector<field> & fields,
- const std::string & default_sorting_field, const size_t num_indices);
+ const std::string & default_sorting_field, const size_t num_indices, const float max_memory_ratio);
~Collection();
@@ -292,5 +294,7 @@ public:
void batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,
size_t &num_indexed);
+
+ bool is_exceeding_memory_threshold() const;
};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index afaa0630..c35c937b 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -35,6 +35,8 @@ private:
size_t default_num_indices;
+ float max_memory_ratio;
+
CollectionManager();
~CollectionManager() = default;
@@ -58,7 +60,8 @@ public:
CollectionManager(CollectionManager const&) = delete;
void operator=(CollectionManager const&) = delete;
- void init(Store *store, const size_t default_num_indices, const std::string & auth_key);
+ void init(Store *store, const size_t default_num_indices, const float max_memory_ratio,
+ const std::string & auth_key);
Option<bool> load(const size_t init_batch_size=1000);
diff --git a/include/config.h b/include/config.h
index e2cf4c73..9df51978 100644
--- a/include/config.h
+++ b/include/config.h
@@ -31,6 +31,8 @@ private:
size_t indices_per_collection;
+ float max_memory_ratio;
+
std::string config_file;
int config_file_validity;
@@ -42,6 +44,7 @@ public:
this->peering_port = 8107;
this->enable_cors = false;
this->indices_per_collection = 4;
+ this->max_memory_ratio = 1.0f;
}
// setters
@@ -154,6 +157,10 @@ public:
return this->nodes;
}
+ float get_max_memory_ratio() const {
+ return this->max_memory_ratio;
+ }
+
// loaders
std::string get_env(const char *name) {
@@ -205,6 +212,10 @@ public:
std::string enable_cors_str = get_env("TYPESENSE_ENABLE_CORS");
StringUtils::toupper(enable_cors_str);
this->enable_cors = ("TRUE" == enable_cors_str) ? true : false;
+
+ if(!get_env("TYPESENSE_MAX_MEMORY_RATIO").empty()) {
+ this->max_memory_ratio = std::stof(get_env("TYPESENSE_MAX_MEMORY_RATIO"));
+ }
}
void load_config_file(cmdline::parser & options) {
@@ -286,6 +297,10 @@ public:
if(reader.Exists("server", "nodes")) {
this->nodes = reader.Get("server", "nodes", "");
}
+
+ if(reader.Exists("server", "max-memory-ratio")) {
+ this->max_memory_ratio = (float) reader.GetReal("server", "max-memory-ratio", 1.0f);
+ }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -349,6 +364,10 @@ public:
if(options.exist("nodes")) {
this->nodes = options.get<std::string>("nodes");
}
+
+ if(options.exist("max-memory-ratio")) {
+ this->max_memory_ratio = options.get<float>("max-memory-ratio");
+ }
}
// validation
diff --git a/include/system_metrics.h b/include/system_metrics.h
index 91bac17c..a6ebf14b 100644
--- a/include/system_metrics.h
+++ b/include/system_metrics.h
@@ -33,6 +33,10 @@ struct cpu_stat_t {
class SystemMetrics {
private:
+ const static uint64_t NON_PROC_MEM_UPDATE_INTERVAL_SECONDS = 60;
+ static uint64_t non_proc_mem_last_access;
+ static uint64_t non_proc_mem_bytes;
+
size_t get_idle_time(const cpu_data_t &e) {
return e.times[S_IDLE] +
e.times[S_IOWAIT];
@@ -117,26 +121,29 @@ private:
}
}
- static unsigned long linux_get_mem_available_bytes() {
- std::string token;
- std::ifstream file("/proc/meminfo");
- while(file >> token) {
- if(token == "MemAvailable:") {
- unsigned long mem_kb;
- if(file >> mem_kb) {
- return mem_kb * 1000;
- } else {
- return 0;
- }
- }
- }
+ static uint64_t get_memory_total_bytes();
- return 0; // nothing found
- }
+ static uint64_t get_memory_used_bytes();
+
+ static uint64_t linux_get_mem_available_bytes();
+
+ static uint64_t get_memory_active_bytes();
+
+ static uint64_t get_memory_non_proc_bytes();
public:
+
+ SystemMetrics() {
+ non_proc_mem_last_access = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+ uint64_t memory_used_bytes = get_memory_used_bytes();
+ non_proc_mem_bytes = memory_used_bytes - get_memory_active_bytes();
+ }
+
void get(const std::string & data_dir_path, nlohmann::json& result);
+ static float used_memory_ratio();
+
std::vector<cpu_stat_t> get_cpu_stats() {
std::vector<cpu_data_t> cpu_data1;
std::vector<cpu_data_t> cpu_data2;
diff --git a/src/collection.cpp b/src/collection.cpp
index b44e15ee..e5495630 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -10,6 +10,7 @@
#include <future>
#include <chrono>
#include <rocksdb/write_batch.h>
+#include <system_metrics.h>
#include "topster.h"
#include "logger.h"
@@ -36,9 +37,12 @@ struct match_index_t {
Collection::Collection(const std::string name, const uint32_t collection_id, const uint64_t created_at,
const uint32_t next_seq_id, Store *store, const std::vector<field> &fields,
- const std::string & default_sorting_field, const size_t num_indices):
+ const std::string & default_sorting_field, const size_t num_indices,
+ const float max_memory_ratio):
name(name), collection_id(collection_id), next_seq_id(next_seq_id), store(store),
- fields(fields), default_sorting_field(default_sorting_field), num_indices(num_indices) {
+ fields(fields), default_sorting_field(default_sorting_field),
+ num_indices(num_indices),
+ max_memory_ratio(max_memory_ratio) {
for(const field& field: fields) {
search_schema.emplace(field.name, field);
@@ -155,6 +159,10 @@ Option<nlohmann::json> Collection::add(const std::string & json_str) {
return Option<nlohmann::json>(doc_seq_id_op.code(), doc_seq_id_op.error());
}
+ if(is_exceeding_memory_threshold()) {
+ return Option<nlohmann::json>(403, "Max memory ratio exceeded.");
+ }
+
const uint32_t seq_id = doc_seq_id_op.get();
const std::string seq_id_str = std::to_string(seq_id);
@@ -182,6 +190,8 @@ Option<nlohmann::json> Collection::add_many(std::vector<std::string>& json_lines
return Option<nlohmann::json>(400, "The request body was empty. So, no records were imported.");
}
+ LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();
+
std::vector<std::vector<index_record>> iter_batch;
for(size_t i = 0; i < num_indices; i++) {
@@ -190,6 +200,7 @@ Option<nlohmann::json> Collection::add_many(std::vector<std::string>& json_lines
const size_t index_batch_size = 1000;
size_t num_indexed = 0;
+ bool exceeds_memory_limit = false;
for(size_t i=0; i < json_lines.size(); i++) {
const std::string & json_line = json_lines[i];
@@ -207,6 +218,21 @@ Option<nlohmann::json> Collection::add_many(std::vector<std::string>& json_lines
continue;
}
+ // check for memory threshold before allowing subsequent batches
+ if(is_exceeding_memory_threshold()) {
+ exceeds_memory_limit = true;
+ }
+
+ if(exceeds_memory_limit) {
+ nlohmann::json index_res;
+ index_res["error"] = "Max memory ratio exceeded.";
+ index_res["success"] = false;
+
+ // NOTE: we overwrite the input json_lines with result to avoid memory pressure
+ json_lines[i] = index_res.dump();
+ continue;
+ }
+
const uint32_t seq_id = doc_seq_id_op.get();
index_record record(i, seq_id, document);
iter_batch[seq_id % this->get_num_indices()].emplace_back(record);
@@ -226,6 +252,10 @@ Option<nlohmann::json> Collection::add_many(std::vector<std::string>& json_lines
return Option<nlohmann::json>(resp_summary);
}
+bool Collection::is_exceeding_memory_threshold() const {
+ return SystemMetrics::used_memory_ratio() > max_memory_ratio;
+}
+
void Collection::batch_index(std::vector<std::vector<index_record>> &index_batches, std::vector<std::string>& json_out,
size_t &num_indexed) {
std::vector<size_t> indexed_counts;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index f5f408d3..d929671f 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -38,7 +38,8 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
store,
fields,
default_sorting_field,
- default_num_indices);
+ default_num_indices,
+ max_memory_ratio);
return collection;
}
@@ -50,10 +51,12 @@ void CollectionManager::add_to_collections(Collection* collection) {
void CollectionManager::init(Store *store,
const size_t default_num_indices,
+ const float max_memory_ratio,
const std::string & auth_key) {
this->store = store;
this->bootstrap_auth_key = auth_key;
this->default_num_indices = default_num_indices;
+ this->max_memory_ratio = max_memory_ratio;
auth_manager.init(store);
}
@@ -271,7 +274,8 @@ Option<Collection*> CollectionManager::create_collection(const std::string name,
collection_meta[COLLECTION_CREATED] = created_at;
Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,
- default_sorting_field, this->default_num_indices);
+ default_sorting_field, this->default_num_indices,
+ this->max_memory_ratio);
next_collection_id++;
rocksdb::WriteBatch batch;
diff --git a/src/system_metrics.cpp b/src/system_metrics.cpp
index 4b1d96d4..c7edb7be 100644
--- a/src/system_metrics.cpp
+++ b/src/system_metrics.cpp
@@ -22,6 +22,9 @@
#define impl_mallctl mallctl
#endif
+uint64_t SystemMetrics::non_proc_mem_last_access = 0;
+uint64_t SystemMetrics::non_proc_mem_bytes = 0;
+
void SystemMetrics::get(const std::string &data_dir_path, nlohmann::json &result) {
// DISK METRICS
struct statvfs st{};
@@ -61,8 +64,58 @@ void SystemMetrics::get(const std::string &data_dir_path, nlohmann::json &result
std::string frag_ratio = format_dp(1.0f - ((float)allocated / active));
result["typesense_memory_fragmentation_ratio"] = frag_ratio;
+ result["system_memory_total_bytes"] = std::to_string(get_memory_total_bytes());
+ result["system_memory_used_bytes"] = std::to_string(get_memory_used_bytes());
+
+ // CPU METRICS
+#if __linux__
+ const std::vector<cpu_stat_t>& cpu_stats = get_cpu_stats();
+
+ for(size_t i = 0; i < cpu_stats.size(); i++) {
+ std::string cpu_id = (i == 0) ? "" : std::to_string(i);
+ result["system_cpu" + cpu_id + "_active_percentage"] = cpu_stats[i].active;
+ }
+#endif
+}
+
+float SystemMetrics::used_memory_ratio() {
+ // non process memory bytes is updated only periodically since it's expensive
+ uint64_t memory_consumed_bytes = get_memory_active_bytes() + get_memory_non_proc_bytes();
+ uint64_t memory_total_bytes = get_memory_total_bytes();
+
+ return ((float)memory_consumed_bytes / memory_total_bytes);
+}
+
+uint64_t SystemMetrics::linux_get_mem_available_bytes() {
+ std::string token;
+ std::ifstream file("/proc/meminfo");
+ while(file >> token) {
+ if(token == "MemAvailable:") {
+ uint64_t mem_kB;
+ if(file >> mem_kB) {
+ return mem_kB * 1024;
+ } else {
+ return 0;
+ }
+ }
+ }
+
+ return 0; // nothing found
+}
+
+uint64_t SystemMetrics::get_memory_active_bytes() {
+ size_t sz, memory_active_bytes = 1;
+ sz = sizeof(size_t);
+ uint64_t epoch = 1;
+
+ impl_mallctl("epoch", &epoch, &sz, &epoch, sz);
+ impl_mallctl("stats.active", &memory_active_bytes, &sz, nullptr, 0);
+ return memory_active_bytes;
+}
+
+uint64_t SystemMetrics::get_memory_used_bytes() {
+ uint64_t memory_total_bytes = get_memory_total_bytes();
uint64_t memory_available_bytes = 0;
- uint64_t memory_total_bytes = 0;
#ifdef __APPLE__
vm_size_t mach_page_size;
@@ -74,30 +127,47 @@ void SystemMetrics::get(const std::string &data_dir_path, nlohmann::json &result
if (KERN_SUCCESS == host_page_size(mach_port, &mach_page_size) &&
KERN_SUCCESS == host_statistics64(mach_port, HOST_VM_INFO,
(host_info64_t)&vm_stats, &count)) {
- memory_available_bytes = (int64_t)(vm_stats.free_count) * (int64_t)mach_page_size;
+ memory_available_bytes = memory_total_bytes -
+ ((int64_t)(vm_stats.active_count + vm_stats.wire_count) * (int64_t)mach_page_size);
}
+#elif __linux__
+ struct sysinfo sys_info;
+ sysinfo(&sys_info);
+ memory_available_bytes = linux_get_mem_available_bytes();
+#endif
+
+ return memory_available_bytes;
+}
+uint64_t SystemMetrics::get_memory_total_bytes() {
+ uint64_t memory_total_bytes = 0;
+
+#ifdef __APPLE__
uint64_t pages = sysconf(_SC_PHYS_PAGES);
uint64_t page_size = sysconf(_SC_PAGE_SIZE);
memory_total_bytes = (pages * page_size);
#elif __linux__
struct sysinfo sys_info;
sysinfo(&sys_info);
- memory_available_bytes = linux_get_mem_available_bytes();
memory_total_bytes = sys_info.totalram;
#endif
- uint64_t memory_used_bytes = memory_total_bytes - memory_available_bytes;
- result["system_memory_used_bytes"] = std::to_string(memory_used_bytes);
- result["system_memory_total_bytes"] = std::to_string(memory_total_bytes);
+ return memory_total_bytes;
+}
- // CPU METRICS
-#if __linux__
- const std::vector<cpu_stat_t>& cpu_stats = get_cpu_stats();
+uint64_t SystemMetrics::get_memory_non_proc_bytes() {
+ uint64_t now = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
- for(size_t i = 0; i < cpu_stats.size(); i++) {
- std::string cpu_id = (i == 0) ? "" : std::to_string(i);
- result["system_cpu" + cpu_id + "_active_percentage"] = cpu_stats[i].active;
+ uint64_t seconds_since_last = (now - non_proc_mem_last_access);
+
+ if(seconds_since_last > NON_PROC_MEM_UPDATE_INTERVAL_SECONDS) {
+ uint64_t memory_used_bytes = get_memory_used_bytes();
+ non_proc_mem_bytes = memory_used_bytes - get_memory_active_bytes();
}
-#endif
+
+ non_proc_mem_last_access = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ return non_proc_mem_bytes;
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 52d4a176..99af2e01 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -130,6 +130,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add("enable-cors", '\0', "Enable CORS requests.");
+ options.add<float>("max-memory-ratio", '\0', "Maximum fraction of system memory to be used.", false, 1.0f);
+
options.add<std::string>("log-dir", '\0', "Path to the log directory.", false, "");
options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");
@@ -371,7 +373,7 @@ int run_server(const Config & config, const std::string & version, void (*master
Store store(db_dir);
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, config.get_indices_per_collection(),
- config.get_api_key());
+ config.get_max_memory_ratio(), config.get_api_key());
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 2df584c4..84ae0f67 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, "auth_key");
+ collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.load();
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 9d02e0a8..b680cc8b 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -18,7 +18,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, "auth_key");
+ collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.load();
std::vector<field> fields = {
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 4c8ecfbb..eed11192 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, "auth_key");
+ collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.load();
search_fields = {
@@ -212,7 +212,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
- collectionManager2.init(store, 4, "auth_key");
+ collectionManager2.init(store, 4, 1.0, "auth_key");
collectionManager2.load();
collection1 = collectionManager2.get_collection("collection1");
@@ -280,7 +280,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *store = new Store(state_dir_path);
- cmanager.init(store, 4, "auth_key");
+ cmanager.init(store, 4, 1.0, "auth_key");
cmanager.load();
// try resolving on a blank slate
@@ -346,7 +346,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
// should be able to restore state on init
CollectionManager & cmanager2 = CollectionManager::get_instance();
- cmanager2.init(store, 4, "auth_key");
+ cmanager2.init(store, 4, 1.0, "auth_key");
cmanager2.load();
collection_option = cmanager2.resolve_symlink("company");
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index e08f32d2..ccc9997e 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -18,7 +18,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, "auth_key");
+ collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.load();
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index e552f307..900612f1 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, "auth_key");
+ collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.load();
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index df6c2d36..f22a5068 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 4, "auth_key");
+ collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.load();
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");