Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add background thread for periodic housekeeping.
Kishore Nallan59f7bf0f4847
diff --git a/WORKSPACE b/WORKSPACEindex f2d86794..80f643c9 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -179,7 +179,7 @@ new_git_repository(new_git_repository(name = "hnsw",build_file = "//bazel:hnsw.BUILD",- commit = "5aba40d4b10dd77aece2ab9a1b3fdf06e433466a",+ commit = "5100d3fe41da45601875b3f395f508398cb12b8a",remote = "https://github.com/typesense/hnswlib.git",)diff --git a/include/collection.h b/include/collection.hindex 112063fc..a48200a7 100644--- a/include/collection.h+++ b/include/collection.h@@ -51,6 +51,8 @@ private:mutable std::shared_mutex mutex;+ mutable std::shared_mutex index_repair_lock;+const uint8_t CURATED_RECORD_IDENTIFIER = 100;const size_t DEFAULT_TOPSTER_SIZE = 250;@@ -432,6 +434,8 @@ public:tsl::htrie_set<char>& include_fields_full,tsl::htrie_set<char>& exclude_fields_full) const;+ void do_housekeeping();+Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,const std::string & filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,diff --git a/include/collection_manager.h b/include/collection_manager.hindex b92f29e5..b11d4c1e 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -126,6 +126,8 @@ public:std::vector<Collection*> get_collections() const;+ std::vector<std::string> get_collection_names() const;+Collection* get_collection_unsafe(const std::string & collection_name) const;// PUBLICLY EXPOSED APIdiff --git a/include/index.h b/include/index.hindex afc2916d..1d895c02 100644--- a/include/index.h+++ b/include/index.h@@ -288,6 +288,9 @@ struct hnsw_index_t {size_t num_dim;vector_distance_type_t distance_type;+ // ensures that this index is not dropped when it's being repaired+ std::mutex repair_m;+hnsw_index_t(size_t num_dim, size_t init_size, vector_distance_type_t distance_type):space(new hnswlib::InnerProductSpace(num_dim)),vecdex(new hnswlib::HierarchicalNSW<float>(space, init_size, 16, 200, 100, true)),@@ -1017,6 +1020,8 @@ public:Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,std::vector<uint32_t>& outside_seq_ids);++ void repair_hnsw_index();};template<class T>diff --git a/include/tsconfig.h b/include/tsconfig.hindex dc382e80..5e754d84 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -69,6 +69,8 @@ private:uint32_t analytics_flush_interval;+ uint32_t housekeeping_interval;+protected:Config() {@@ -96,6 +98,7 @@ protected:this->enable_search_analytics = false;this->analytics_flush_interval = 3600; // in seconds+ this->housekeeping_interval = 1800; // in seconds}Config(Config const&) {@@ -294,6 +297,10 @@ public:return this->analytics_flush_interval;}+ size_t get_housekeeping_interval() const {+ return this->housekeeping_interval;+ }+size_t get_thread_pool_size() const {return this->thread_pool_size;}@@ -429,6 +436,10 @@ public:this->analytics_flush_interval = std::stoi(get_env("TYPESENSE_ANALYTICS_FLUSH_INTERVAL"));}+ if(!get_env("TYPESENSE_HOUSEKEEPING_INTERVAL").empty()) {+ this->housekeeping_interval = std::stoi(get_env("TYPESENSE_HOUSEKEEPING_INTERVAL"));+ }+if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));}@@ -592,6 +603,10 @@ public:this->analytics_flush_interval = (int) reader.GetInteger("server", "analytics-flush-interval", 3600);}+ if(reader.Exists("server", "housekeeping-interval")) {+ this->housekeeping_interval = (int) reader.GetInteger("server", "housekeeping-interval", 1800);+ }+if(reader.Exists("server", "thread-pool-size")) {this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);}@@ -746,6 +761,10 @@ public:this->analytics_flush_interval = options.get<uint32_t>("analytics-flush-interval");}+ if(options.exist("housekeeping-interval")) {+ this->housekeeping_interval = options.get<uint32_t>("housekeeping-interval");+ }+if(options.exist("thread-pool-size")) {this->thread_pool_size = options.get<uint32_t>("thread-pool-size");}diff --git a/src/collection.cpp b/src/collection.cppindex 7ea538a8..29a17688 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -58,6 +58,7 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, coCollection::~Collection() {std::unique_lock lock(mutex);+ std::unique_lock repair_lock(index_repair_lock);delete index;delete synonym_index;}@@ -5208,3 +5209,8 @@ void Collection::remove_embedding_field(const std::string& field_name) {tsl::htrie_map<char, field> Collection::get_embedding_fields_unsafe() {return embedding_fields;}++void Collection::do_housekeeping() {+ std::unique_lock lock(index_repair_lock);+ index->repair_hnsw_index();+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 5e8d0f88..236dd22a 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -473,6 +473,17 @@ std::vector<Collection*> CollectionManager::get_collections() const {return collection_vec;}+std::vector<std::string> CollectionManager::get_collection_names() const {+ std::shared_lock lock(mutex);++ std::vector<std::string> collection_vec;+ for(const auto& kv: collections) {+ collection_vec.push_back(kv.first);+ }++ return collection_vec;+}+Option<nlohmann::json> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {std::shared_lock s_lock(mutex);auto collection = get_collection_unsafe(collection_name);diff --git a/src/index.cpp b/src/index.cppindex 5a362f52..f883c127 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -206,6 +206,7 @@ Index::~Index() {delete seq_ids;for(auto& vec_index_kv: vector_index) {+ std::unique_lock lock(vec_index_kv.second->repair_m);delete vec_index_kv.second;}}@@ -6372,6 +6373,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vecif(del_field.num_dim) {auto hnsw_index = vector_index[del_field.name];+ std::unique_lock lock(hnsw_index->repair_m);delete hnsw_index;vector_index.erase(del_field.name);}@@ -6749,6 +6751,31 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,}}+void Index::repair_hnsw_index() {+ std::vector<std::string> vector_fields;++ // this lock ensures that the `vector_index` map is not mutated during read+ std::shared_lock read_lock(mutex);++ for(auto& vec_kv: vector_index) {+ vector_fields.push_back(vec_kv.first);+ }++ read_lock.unlock();++ for(const auto& vector_field: vector_fields) {+ read_lock.lock();+ if(vector_index.count(vector_field) != 0) {+ // this lock ensures that the vector index is not dropped during repair+ std::unique_lock lock(vector_index[vector_field]->repair_m);+ read_lock.unlock(); // release this lock since repair is a long running operation+ vector_index[vector_field]->vecdex->repair_zero_indegree();+ } else {+ read_lock.unlock();+ }+ }+}+/*// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex c628c569..195db3f2 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -13,7 +13,8 @@#include <arpa/inet.h>#include <sys/socket.h>#include <ifaddrs.h>-#include <analytics_manager.h>+#include "analytics_manager.h"+#include "housekeeper.h"#include "core_api.h"#include "ratelimit_manager.h"@@ -103,6 +104,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);options.add<int>("cache-num-entries", '\0', "Number of entries to cache.", false, 1000);options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);+ options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");@@ -453,6 +455,11 @@ int run_server(const Config & config, const std::string & version, void (*masterAnalyticsManager::get_instance().run(&replication_state);});+ HouseKeeper::get_instance().init(config.get_housekeeping_interval());+ std::thread housekeeping_thread([]() {+ HouseKeeper::get_instance().run();+ });+RemoteEmbedder::init(&replication_state);std::string path_to_nodes = config.get_nodes();@@ -477,6 +484,10 @@ int run_server(const Config & config, const std::string & version, void (*masterLOG(INFO) << "Waiting for event sink thread to be done...";event_sink_thread.join();+ LOG(INFO) << "Waiting for housekeeping thread to be done...";+ HouseKeeper::get_instance().stop();+ housekeeping_thread.join();+LOG(INFO) << "Shutting down server_thread_pool";server_thread_pool.shutdown();