Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Expose system metrics via API.

kishorencbbc9c9780d02
diff --git a/docker/deployment.Dockerfile b/docker/deployment.Dockerfile
index be92a5e6..1948a4b3 100644
--- a/docker/deployment.Dockerfile
+++ b/docker/deployment.Dockerfile
@@ -1,5 +1,7 @@
FROM ubuntu:16.04
+RUN apt-get -y update && apt-get -y install ca-certificates
+
RUN mkdir -p /opt
COPY typesense-server /opt
RUN chmod +x /opt/typesense-server
diff --git a/include/core_api.h b/include/core_api.h
index f2c227ea..b16565e9 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -14,6 +14,8 @@ bool get_debug(http_req & req, http_res & res);
bool get_health(http_req & req, http_res & res);
+bool get_metrics_json(http_req & req, http_res & res);
+
bool get_search(http_req & req, http_res & res);
bool get_collection_summary(http_req & req, http_res & res);
diff --git a/include/system_metrics.h b/include/system_metrics.h
new file mode 100644
index 00000000..fda58fdd
--- /dev/null
+++ b/include/system_metrics.h
@@ -0,0 +1,140 @@
+#include <string>
+#include <fstream>
+#include <sstream>
+#include <thread>
+#include "json.hpp"
+
+const int NUM_CPU_STATES = 10;
+
+struct cpu_data_t {
+ std::string cpu;
+ size_t times[NUM_CPU_STATES];
+};
+
+enum CPUStates {
+ S_USER = 0,
+ S_NICE,
+ S_SYSTEM,
+ S_IDLE,
+ S_IOWAIT,
+ S_IRQ,
+ S_SOFTIRQ,
+ S_STEAL,
+ S_GUEST,
+ S_GUEST_NICE
+};
+
+struct cpu_stat_t {
+ std::string active;
+ std::string idle;
+};
+
+class SystemMetrics {
+private:
+
+ size_t get_idle_time(const cpu_data_t &e) {
+ return e.times[S_IDLE] +
+ e.times[S_IOWAIT];
+ }
+
+ size_t get_active_time(const cpu_data_t &e) {
+ return e.times[S_USER] +
+ e.times[S_NICE] +
+ e.times[S_SYSTEM] +
+ e.times[S_IRQ] +
+ e.times[S_SOFTIRQ] +
+ e.times[S_STEAL] +
+ e.times[S_GUEST] +
+ e.times[S_GUEST_NICE];
+ }
+
+ std::vector<cpu_stat_t> compute_cpu_stats(const std::vector<cpu_data_t>& cpu_data1,
+ const std::vector<cpu_data_t>& cpu_data2) {
+ std::vector<cpu_stat_t> stats;
+ const size_t NUM_ENTRIES = cpu_data1.size();
+
+ for (size_t i = 0; i < NUM_ENTRIES; ++i) {
+ cpu_stat_t stat;
+
+ const cpu_data_t &d1 = cpu_data1[i];
+ const cpu_data_t &d2 = cpu_data2[i];
+
+ const float active_time = static_cast<float>(get_active_time(d2) - get_active_time(d1));
+ const float idle_time = static_cast<float>(get_idle_time(d2) - get_idle_time(d1));
+ const float total_time = active_time + idle_time;
+
+ float active_percentage = 100.f * (active_time / total_time);
+ float idle_percentage = 100.f * (idle_time / total_time);
+
+ std::stringstream active_ss;
+ active_ss.setf(std::ios::fixed, std::ios::floatfield);
+ active_ss.precision(2);
+ active_ss << active_percentage;
+ stat.active = active_ss.str();
+
+ std::stringstream idle_ss;
+ idle_ss.setf(std::ios::fixed, std::ios::floatfield);
+ idle_ss.precision(2);
+ idle_ss << idle_percentage;
+ stat.idle = idle_ss.str();
+
+ stats.push_back(stat);
+ }
+
+ return stats;
+ }
+
+ void read_cpu_data(std::vector<cpu_data_t> &entries) {
+ std::ifstream stat_file("/proc/stat");
+
+ std::string line;
+
+ const std::string STR_CPU("cpu");
+ const std::string STR_TOT("tot");
+
+ while (std::getline(stat_file, line)) {
+ // cpu stats line found
+ if (!line.compare(0, STR_CPU.size(), STR_CPU)) {
+ std::istringstream ss(line);
+
+ // store entry
+ entries.emplace_back(cpu_data_t());
+ cpu_data_t &entry = entries.back();
+
+ // read cpu label
+ ss >> entry.cpu;
+
+ if (entry.cpu.size() > STR_CPU.size()) {
+ entry.cpu.erase(0, STR_CPU.size());
+ } else {
+ entry.cpu = STR_TOT;
+ }
+
+ // read times
+ for (int i = 0; i < NUM_CPU_STATES; ++i) {
+ ss >> entry.times[i];
+ }
+ }
+ }
+ }
+
+public:
+ void get(const std::string & data_dir_path, nlohmann::json& result);
+
+ std::vector<cpu_stat_t> get_cpu_stats() {
+ std::vector<cpu_data_t> cpu_data1;
+ std::vector<cpu_data_t> cpu_data2;
+
+ // snapshot 1
+ read_cpu_data(cpu_data1);
+
+ // 100ms pause
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+ // snapshot 2
+ read_cpu_data(cpu_data2);
+
+ // compute
+ return compute_cpu_stats(cpu_data1, cpu_data2);
+ }
+};
\ No newline at end of file
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 8f5a1e83..7f1f2d21 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1,12 +1,12 @@
#include <regex>
#include <chrono>
#include <thread>
-#include <sys/resource.h>
#include "typesense_server_utils.h"
#include "core_api.h"
#include "string_utils.h"
#include "collection.h"
#include "collection_manager.h"
+#include "system_metrics.h"
#include "logger.h"
nlohmann::json collection_summary_json(Collection *collection) {
@@ -187,7 +187,7 @@ bool get_debug(http_req & req, http_res & res) {
uint64_t state = server->node_state();
result["state"] = state;
-
+
res.set_200(result.dump());
return true;
}
@@ -205,6 +205,20 @@ bool get_health(http_req & req, http_res & res) {
return alive;
}
+
+bool get_metrics_json(http_req &req, http_res &res) {
+ nlohmann::json result;
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ const std::string & data_dir_path = collectionManager.get_store()->get_state_dir_path();
+
+ SystemMetrics sys_metrics;
+ sys_metrics.get(data_dir_path, result);
+
+ res.set_body(200, result.dump(2));
+ return true;
+}
+
bool get_search(http_req & req, http_res & res) {
auto begin = std::chrono::high_resolution_clock::now();
diff --git a/src/http_client.cpp b/src/http_client.cpp
index ea366009..bdaed66d 100644
--- a/src/http_client.cpp
+++ b/src/http_client.cpp
@@ -49,12 +49,12 @@ void HttpClient::init(const std::string &api_key) {
// try to locate ca cert file (from: https://serverfault.com/a/722646/117601)
std::vector<std::string> locations = {
- "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
- "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
- "/etc/ssl/ca-bundle.pem", // OpenSUSE
- "/etc/pki/tls/cacert.pem", // OpenELEC
- "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
- "/usr/local/etc/openssl/cert.pem", // OSX
+ "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
+ "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
+ "/etc/ssl/ca-bundle.pem", // OpenSUSE
+ "/etc/pki/tls/cacert.pem", // OpenELEC
+ "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
+ "/usr/local/etc/openssl/cert.pem", // OSX
};
HttpClient::ca_cert_path = "";
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index f4f0e110..7281de86 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -20,6 +20,7 @@ void master_server_routes() {
server->del("/collections/:collection/documents/:id", del_remove_document);
// meta
+ server->get("/metrics.json", get_metrics_json);
server->get("/debug", get_debug);
server->get("/health", get_health);
}
diff --git a/src/system_metrics.cpp b/src/system_metrics.cpp
new file mode 100644
index 00000000..b3f1fa81
--- /dev/null
+++ b/src/system_metrics.cpp
@@ -0,0 +1,71 @@
+#include "system_metrics.h"
+
+#include <sys/resource.h>
+#include <sys/statvfs.h>
+#if __linux__
+#include <sys/types.h>
+#include <sys/sysinfo.h>
+#include <unistd.h>
+#elif __APPLE__
+#include <unistd.h>
+#include <mach/vm_statistics.h>
+#include <mach/mach_types.h>
+#include <mach/mach_init.h>
+#include <mach/mach_host.h>
+#endif
+
+void SystemMetrics::get(const std::string &data_dir_path, nlohmann::json &result) {
+ // DISK METRICS
+ struct statvfs st{};
+ statvfs(data_dir_path.c_str(), &st);
+ uint64_t disk_total_bytes = st.f_blocks * st.f_frsize;
+ uint64_t disk_used_bytes = (st.f_blocks - st.f_bavail) * st.f_frsize;
+ result["disk_total_bytes"] = disk_total_bytes;
+ result["disk_used_bytes"] = disk_used_bytes;
+
+ // MEMORY METRICS
+
+ rusage r_usage;
+ getrusage(RUSAGE_SELF, &r_usage);
+ result["memory_used_process_bytes"] = r_usage.ru_maxrss;
+
+ uint64_t memory_free_bytes = 0;
+ uint64_t memory_total_bytes = 0;
+
+#ifdef __APPLE__
+ vm_size_t mach_page_size;
+ mach_port_t mach_port;
+ mach_msg_type_number_t count;
+ vm_statistics64_data_t vm_stats;
+ mach_port = mach_host_self();
+ count = sizeof(vm_stats) / sizeof(natural_t);
+ if (KERN_SUCCESS == host_page_size(mach_port, &mach_page_size) &&
+ KERN_SUCCESS == host_statistics64(mach_port, HOST_VM_INFO,
+ (host_info64_t)&vm_stats, &count)) {
+ memory_free_bytes = (int64_t)(vm_stats.free_count) * (int64_t)mach_page_size;
+ }
+
+ uint64_t pages = sysconf(_SC_PHYS_PAGES);
+ uint64_t page_size = sysconf(_SC_PAGE_SIZE);
+ memory_total_bytes = (pages * page_size);
+#elif __linux__
+ struct sysinfo sys_info;
+ sysinfo(&sys_info);
+ memory_free_bytes = sys_info.freeram;
+ memory_total_bytes = sys_info.totalram;
+#endif
+
+ result["memory_free_bytes"] = memory_free_bytes;
+ result["memory_total_bytes"] = memory_total_bytes;
+
+ // CPU METRICS
+#if __linux__
+ const std::vector<cpu_stat_t>& cpu_stats = get_cpu_stats();
+
+ for(size_t i = 0; i < cpu_stats.size(); i++) {
+ std::string cpu_label = std::to_string(i+1);
+ result["cpu" + cpu_label + "_active_percentage"] = cpu_stats[i].active;
+ result["cpu" + cpu_label + "_idle_percentage"] = cpu_stats[i].idle;
+ }
+#endif
+}