Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Expose system metrics via API.
kishorencbbc9c9780d02
diff --git a/docker/deployment.Dockerfile b/docker/deployment.Dockerfileindex be92a5e6..1948a4b3 100644--- a/docker/deployment.Dockerfile+++ b/docker/deployment.Dockerfile@@ -1,5 +1,7 @@FROM ubuntu:16.04+RUN apt-get -y update && apt-get -y install ca-certificates+RUN mkdir -p /optCOPY typesense-server /optRUN chmod +x /opt/typesense-serverdiff --git a/include/core_api.h b/include/core_api.hindex f2c227ea..b16565e9 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -14,6 +14,8 @@ bool get_debug(http_req & req, http_res & res);bool get_health(http_req & req, http_res & res);+bool get_metrics_json(http_req & req, http_res & res);+bool get_search(http_req & req, http_res & res);bool get_collection_summary(http_req & req, http_res & res);diff --git a/include/system_metrics.h b/include/system_metrics.hnew file mode 100644index 00000000..fda58fdd--- /dev/null+++ b/include/system_metrics.h@@ -0,0 +1,140 @@+#include <string>+#include <fstream>+#include <sstream>+#include <thread>+#include "json.hpp"++const int NUM_CPU_STATES = 10;++struct cpu_data_t {+ std::string cpu;+ size_t times[NUM_CPU_STATES];+};++enum CPUStates {+ S_USER = 0,+ S_NICE,+ S_SYSTEM,+ S_IDLE,+ S_IOWAIT,+ S_IRQ,+ S_SOFTIRQ,+ S_STEAL,+ S_GUEST,+ S_GUEST_NICE+};++struct cpu_stat_t {+ std::string active;+ std::string idle;+};++class SystemMetrics {+private:++ size_t get_idle_time(const cpu_data_t &e) {+ return e.times[S_IDLE] ++ e.times[S_IOWAIT];+ }++ size_t get_active_time(const cpu_data_t &e) {+ return e.times[S_USER] ++ e.times[S_NICE] ++ e.times[S_SYSTEM] ++ e.times[S_IRQ] ++ e.times[S_SOFTIRQ] ++ e.times[S_STEAL] ++ e.times[S_GUEST] ++ e.times[S_GUEST_NICE];+ }++ std::vector<cpu_stat_t> compute_cpu_stats(const std::vector<cpu_data_t>& cpu_data1,+ const std::vector<cpu_data_t>& cpu_data2) {+ std::vector<cpu_stat_t> stats;+ const size_t NUM_ENTRIES = cpu_data1.size();++ for (size_t i = 0; i < NUM_ENTRIES; ++i) {+ cpu_stat_t stat;++ const cpu_data_t &d1 = cpu_data1[i];+ const cpu_data_t &d2 = cpu_data2[i];++ const float active_time = static_cast<float>(get_active_time(d2) - get_active_time(d1));+ const float idle_time = static_cast<float>(get_idle_time(d2) - get_idle_time(d1));+ const float total_time = active_time + idle_time;++ float active_percentage = 100.f * (active_time / total_time);+ float idle_percentage = 100.f * (idle_time / total_time);++ std::stringstream active_ss;+ active_ss.setf(std::ios::fixed, std::ios::floatfield);+ active_ss.precision(2);+ active_ss << active_percentage;+ stat.active = active_ss.str();++ std::stringstream idle_ss;+ idle_ss.setf(std::ios::fixed, std::ios::floatfield);+ idle_ss.precision(2);+ idle_ss << idle_percentage;+ stat.idle = idle_ss.str();++ stats.push_back(stat);+ }++ return stats;+ }++ void read_cpu_data(std::vector<cpu_data_t> &entries) {+ std::ifstream stat_file("/proc/stat");++ std::string line;++ const std::string STR_CPU("cpu");+ const std::string STR_TOT("tot");++ while (std::getline(stat_file, line)) {+ // cpu stats line found+ if (!line.compare(0, STR_CPU.size(), STR_CPU)) {+ std::istringstream ss(line);++ // store entry+ entries.emplace_back(cpu_data_t());+ cpu_data_t &entry = entries.back();++ // read cpu label+ ss >> entry.cpu;++ if (entry.cpu.size() > STR_CPU.size()) {+ entry.cpu.erase(0, STR_CPU.size());+ } else {+ entry.cpu = STR_TOT;+ }++ // read times+ for (int i = 0; i < NUM_CPU_STATES; ++i) {+ ss >> entry.times[i];+ }+ }+ }+ }++public:+ void get(const std::string & data_dir_path, nlohmann::json& result);++ std::vector<cpu_stat_t> get_cpu_stats() {+ std::vector<cpu_data_t> cpu_data1;+ std::vector<cpu_data_t> cpu_data2;++ // snapshot 1+ read_cpu_data(cpu_data1);++ // 100ms pause+ std::this_thread::sleep_for(std::chrono::milliseconds(100));++ // snapshot 2+ read_cpu_data(cpu_data2);++ // compute+ return compute_cpu_stats(cpu_data1, cpu_data2);+ }+};\ No newline at end of filediff --git a/src/core_api.cpp b/src/core_api.cppindex 8f5a1e83..7f1f2d21 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1,12 +1,12 @@#include <regex>#include <chrono>#include <thread>-#include <sys/resource.h>#include "typesense_server_utils.h"#include "core_api.h"#include "string_utils.h"#include "collection.h"#include "collection_manager.h"+#include "system_metrics.h"#include "logger.h"nlohmann::json collection_summary_json(Collection *collection) {@@ -187,7 +187,7 @@ bool get_debug(http_req & req, http_res & res) {uint64_t state = server->node_state();result["state"] = state;-+res.set_200(result.dump());return true;}@@ -205,6 +205,20 @@ bool get_health(http_req & req, http_res & res) {return alive;}++bool get_metrics_json(http_req &req, http_res &res) {+ nlohmann::json result;++ CollectionManager & collectionManager = CollectionManager::get_instance();+ const std::string & data_dir_path = collectionManager.get_store()->get_state_dir_path();++ SystemMetrics sys_metrics;+ sys_metrics.get(data_dir_path, result);++ res.set_body(200, result.dump(2));+ return true;+}+bool get_search(http_req & req, http_res & res) {auto begin = std::chrono::high_resolution_clock::now();diff --git a/src/http_client.cpp b/src/http_client.cppindex ea366009..bdaed66d 100644--- a/src/http_client.cpp+++ b/src/http_client.cpp@@ -49,12 +49,12 @@ void HttpClient::init(const std::string &api_key) {// try to locate ca cert file (from: https://serverfault.com/a/722646/117601)std::vector<std::string> locations = {- "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.- "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6- "/etc/ssl/ca-bundle.pem", // OpenSUSE- "/etc/pki/tls/cacert.pem", // OpenELEC- "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7- "/usr/local/etc/openssl/cert.pem", // OSX+ "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.+ "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6+ "/etc/ssl/ca-bundle.pem", // OpenSUSE+ "/etc/pki/tls/cacert.pem", // OpenELEC+ "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7+ "/usr/local/etc/openssl/cert.pem", // OSX};HttpClient::ca_cert_path = "";diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex f4f0e110..7281de86 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -20,6 +20,7 @@ void master_server_routes() {server->del("/collections/:collection/documents/:id", del_remove_document);// meta+ server->get("/metrics.json", get_metrics_json);server->get("/debug", get_debug);server->get("/health", get_health);}diff --git a/src/system_metrics.cpp b/src/system_metrics.cppnew file mode 100644index 00000000..b3f1fa81--- /dev/null+++ b/src/system_metrics.cpp@@ -0,0 +1,71 @@+#include "system_metrics.h"++#include <sys/resource.h>+#include <sys/statvfs.h>+#if __linux__+#include <sys/types.h>+#include <sys/sysinfo.h>+#include <unistd.h>+#elif __APPLE__+#include <unistd.h>+#include <mach/vm_statistics.h>+#include <mach/mach_types.h>+#include <mach/mach_init.h>+#include <mach/mach_host.h>+#endif++void SystemMetrics::get(const std::string &data_dir_path, nlohmann::json &result) {+ // DISK METRICS+ struct statvfs st{};+ statvfs(data_dir_path.c_str(), &st);+ uint64_t disk_total_bytes = st.f_blocks * st.f_frsize;+ uint64_t disk_used_bytes = (st.f_blocks - st.f_bavail) * st.f_frsize;+ result["disk_total_bytes"] = disk_total_bytes;+ result["disk_used_bytes"] = disk_used_bytes;++ // MEMORY METRICS++ rusage r_usage;+ getrusage(RUSAGE_SELF, &r_usage);+ result["memory_used_process_bytes"] = r_usage.ru_maxrss;++ uint64_t memory_free_bytes = 0;+ uint64_t memory_total_bytes = 0;++#ifdef __APPLE__+ vm_size_t mach_page_size;+ mach_port_t mach_port;+ mach_msg_type_number_t count;+ vm_statistics64_data_t vm_stats;+ mach_port = mach_host_self();+ count = sizeof(vm_stats) / sizeof(natural_t);+ if (KERN_SUCCESS == host_page_size(mach_port, &mach_page_size) &&+ KERN_SUCCESS == host_statistics64(mach_port, HOST_VM_INFO,+ (host_info64_t)&vm_stats, &count)) {+ memory_free_bytes = (int64_t)(vm_stats.free_count) * (int64_t)mach_page_size;+ }++ uint64_t pages = sysconf(_SC_PHYS_PAGES);+ uint64_t page_size = sysconf(_SC_PAGE_SIZE);+ memory_total_bytes = (pages * page_size);+#elif __linux__+ struct sysinfo sys_info;+ sysinfo(&sys_info);+ memory_free_bytes = sys_info.freeram;+ memory_total_bytes = sys_info.totalram;+#endif++ result["memory_free_bytes"] = memory_free_bytes;+ result["memory_total_bytes"] = memory_total_bytes;++ // CPU METRICS+#if __linux__+ const std::vector<cpu_stat_t>& cpu_stats = get_cpu_stats();++ for(size_t i = 0; i < cpu_stats.size(); i++) {+ std::string cpu_label = std::to_string(i+1);+ result["cpu" + cpu_label + "_active_percentage"] = cpu_stats[i].active;+ result["cpu" + cpu_label + "_idle_percentage"] = cpu_stats[i].idle;+ }+#endif+}