Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add background thread for periodic housekeeping.

Kishore Nallan59f7bf0f4847
diff --git a/WORKSPACE b/WORKSPACE
index f2d86794..80f643c9 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -179,7 +179,7 @@ new_git_repository(
new_git_repository(
name = "hnsw",
build_file = "//bazel:hnsw.BUILD",
- commit = "5aba40d4b10dd77aece2ab9a1b3fdf06e433466a",
+ commit = "5100d3fe41da45601875b3f395f508398cb12b8a",
remote = "https://github.com/typesense/hnswlib.git",
)
diff --git a/include/collection.h b/include/collection.h
index 112063fc..a48200a7 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -51,6 +51,8 @@ private:
mutable std::shared_mutex mutex;
+ mutable std::shared_mutex index_repair_lock;
+
const uint8_t CURATED_RECORD_IDENTIFIER = 100;
const size_t DEFAULT_TOPSTER_SIZE = 250;
@@ -432,6 +434,8 @@ public:
tsl::htrie_set<char>& include_fields_full,
tsl::htrie_set<char>& exclude_fields_full) const;
+ void do_housekeeping();
+
Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,
const std::string & filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index b92f29e5..b11d4c1e 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -126,6 +126,8 @@ public:
std::vector<Collection*> get_collections() const;
+ std::vector<std::string> get_collection_names() const;
+
Collection* get_collection_unsafe(const std::string & collection_name) const;
// PUBLICLY EXPOSED API
diff --git a/include/index.h b/include/index.h
index afc2916d..1d895c02 100644
--- a/include/index.h
+++ b/include/index.h
@@ -288,6 +288,9 @@ struct hnsw_index_t {
size_t num_dim;
vector_distance_type_t distance_type;
+ // ensures that this index is not dropped when it's being repaired
+ std::mutex repair_m;
+
hnsw_index_t(size_t num_dim, size_t init_size, vector_distance_type_t distance_type):
space(new hnswlib::InnerProductSpace(num_dim)),
vecdex(new hnswlib::HierarchicalNSW<float>(space, init_size, 16, 200, 100, true)),
@@ -1017,6 +1020,8 @@ public:
Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,
std::vector<uint32_t>& outside_seq_ids);
+
+ void repair_hnsw_index();
};
template<class T>
diff --git a/include/tsconfig.h b/include/tsconfig.h
index dc382e80..5e754d84 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -69,6 +69,8 @@ private:
uint32_t analytics_flush_interval;
+ uint32_t housekeeping_interval;
+
protected:
Config() {
@@ -96,6 +98,7 @@ protected:
this->enable_search_analytics = false;
this->analytics_flush_interval = 3600; // in seconds
+ this->housekeeping_interval = 1800; // in seconds
}
Config(Config const&) {
@@ -294,6 +297,10 @@ public:
return this->analytics_flush_interval;
}
+ size_t get_housekeeping_interval() const {
+ return this->housekeeping_interval;
+ }
+
size_t get_thread_pool_size() const {
return this->thread_pool_size;
}
@@ -429,6 +436,10 @@ public:
this->analytics_flush_interval = std::stoi(get_env("TYPESENSE_ANALYTICS_FLUSH_INTERVAL"));
}
+ if(!get_env("TYPESENSE_HOUSEKEEPING_INTERVAL").empty()) {
+ this->housekeeping_interval = std::stoi(get_env("TYPESENSE_HOUSEKEEPING_INTERVAL"));
+ }
+
if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {
this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));
}
@@ -592,6 +603,10 @@ public:
this->analytics_flush_interval = (int) reader.GetInteger("server", "analytics-flush-interval", 3600);
}
+ if(reader.Exists("server", "housekeeping-interval")) {
+ this->housekeeping_interval = (int) reader.GetInteger("server", "housekeeping-interval", 1800);
+ }
+
if(reader.Exists("server", "thread-pool-size")) {
this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);
}
@@ -746,6 +761,10 @@ public:
this->analytics_flush_interval = options.get<uint32_t>("analytics-flush-interval");
}
+ if(options.exist("housekeeping-interval")) {
+ this->housekeeping_interval = options.get<uint32_t>("housekeeping-interval");
+ }
+
if(options.exist("thread-pool-size")) {
this->thread_pool_size = options.get<uint32_t>("thread-pool-size");
}
diff --git a/src/collection.cpp b/src/collection.cpp
index 7ea538a8..29a17688 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -58,6 +58,7 @@ Collection::Collection(const std::string& name, const uint32_t collection_id, co
Collection::~Collection() {
std::unique_lock lock(mutex);
+ std::unique_lock repair_lock(index_repair_lock);
delete index;
delete synonym_index;
}
@@ -5208,3 +5209,8 @@ void Collection::remove_embedding_field(const std::string& field_name) {
tsl::htrie_map<char, field> Collection::get_embedding_fields_unsafe() {
return embedding_fields;
}
+
+void Collection::do_housekeeping() {
+ std::unique_lock lock(index_repair_lock);
+ index->repair_hnsw_index();
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 5e8d0f88..236dd22a 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -473,6 +473,17 @@ std::vector<Collection*> CollectionManager::get_collections() const {
return collection_vec;
}
+std::vector<std::string> CollectionManager::get_collection_names() const {
+ std::shared_lock lock(mutex);
+
+ std::vector<std::string> collection_vec;
+ for(const auto& kv: collections) {
+ collection_vec.push_back(kv.first);
+ }
+
+ return collection_vec;
+}
+
Option<nlohmann::json> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {
std::shared_lock s_lock(mutex);
auto collection = get_collection_unsafe(collection_name);
diff --git a/src/index.cpp b/src/index.cpp
index 5a362f52..f883c127 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -206,6 +206,7 @@ Index::~Index() {
delete seq_ids;
for(auto& vec_index_kv: vector_index) {
+ std::unique_lock lock(vec_index_kv.second->repair_m);
delete vec_index_kv.second;
}
}
@@ -6372,6 +6373,7 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
if(del_field.num_dim) {
auto hnsw_index = vector_index[del_field.name];
+ std::unique_lock lock(hnsw_index->repair_m);
delete hnsw_index;
vector_index.erase(del_field.name);
}
@@ -6749,6 +6751,31 @@ void Index::batch_embed_fields(std::vector<index_record*>& records,
}
}
+void Index::repair_hnsw_index() {
+ std::vector<std::string> vector_fields;
+
+ // this lock ensures that the `vector_index` map is not mutated during read
+ std::shared_lock read_lock(mutex);
+
+ for(auto& vec_kv: vector_index) {
+ vector_fields.push_back(vec_kv.first);
+ }
+
+ read_lock.unlock();
+
+ for(const auto& vector_field: vector_fields) {
+ read_lock.lock();
+ if(vector_index.count(vector_field) != 0) {
+ // this lock ensures that the vector index is not dropped during repair
+ std::unique_lock lock(vector_index[vector_field]->repair_m);
+ read_lock.unlock(); // release this lock since repair is a long running operation
+ vector_index[vector_field]->vecdex->repair_zero_indegree();
+ } else {
+ read_lock.unlock();
+ }
+ }
+}
+
/*
// https://stackoverflow.com/questions/924171/geo-fencing-point-inside-outside-polygon
// NOTE: polygon and point should have been transformed with `transform_for_180th_meridian`
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index c628c569..195db3f2 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -13,7 +13,8 @@
#include <arpa/inet.h>
#include <sys/socket.h>
#include <ifaddrs.h>
-#include <analytics_manager.h>
+#include "analytics_manager.h"
+#include "housekeeper.h"
#include "core_api.h"
#include "ratelimit_manager.h"
@@ -103,6 +104,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);
options.add<int>("cache-num-entries", '\0', "Number of entries to cache.", false, 1000);
options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);
+ options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
@@ -453,6 +455,11 @@ int run_server(const Config & config, const std::string & version, void (*master
AnalyticsManager::get_instance().run(&replication_state);
});
+ HouseKeeper::get_instance().init(config.get_housekeeping_interval());
+ std::thread housekeeping_thread([]() {
+ HouseKeeper::get_instance().run();
+ });
+
RemoteEmbedder::init(&replication_state);
std::string path_to_nodes = config.get_nodes();
@@ -477,6 +484,10 @@ int run_server(const Config & config, const std::string & version, void (*master
LOG(INFO) << "Waiting for event sink thread to be done...";
event_sink_thread.join();
+ LOG(INFO) << "Waiting for housekeeping thread to be done...";
+ HouseKeeper::get_instance().stop();
+ housekeeping_thread.join();
+
LOG(INFO) << "Shutting down server_thread_pool";
server_thread_pool.shutdown();