Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Basics of background replication.

Kishore Nallan58a10877ee23
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6094b8fe..0dea3d4f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -90,7 +90,7 @@ if(NOT APPLE)
list(APPEND ROCKSDB_LIBS rt)
endif()
-target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
-target_link_libraries(benchmark for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(typesense-server h2o-evloop for pthread curl ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(search for pthread h2o-evloop curl ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
+target_link_libraries(benchmark for pthread curl h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
diff --git a/TODO.md b/TODO.md
index 57361200..92f53ef3 100644
--- a/TODO.md
+++ b/TODO.md
@@ -57,6 +57,8 @@
- ~~only last token should be prefix searched~~
- ~~Prefix-search strings should not be null terminated~~
- ~~sort results by float field~~
+- Collection Manager collections map should store plain collection name
+- init_collection of Collection manager should probably take seq_id as param
- Validate before string to int conversion in the http api layer
- When field of "id" but not string, what happens?
- Typo in prefix search
diff --git a/docker/development.Dockerfile b/docker/development.Dockerfile
index 36a3af6d..54f1e4d8 100644
--- a/docker/development.Dockerfile
+++ b/docker/development.Dockerfile
@@ -24,5 +24,7 @@ RUN make -C /opt/openssl-1.0.2k depend
RUN make -C /opt/openssl-1.0.2k -j4
RUN make -C /opt/openssl-1.0.2k install
+RUN apt-get install curl
+
ENV CC /usr/local/gcc-4.9.2/bin/gcc
ENV CXX /usr/local/gcc-4.9.2/bin/g++
diff --git a/include/api.h b/include/api.h
index 1a88181b..f5147edc 100644
--- a/include/api.h
+++ b/include/api.h
@@ -18,4 +18,9 @@ void get_fetch_document(http_req & req, http_res & res);
void del_remove_document(http_req & req, http_res & res);
-void get_replication_updates(http_req &req, http_res &res);
\ No newline at end of file
+void get_replication_updates(http_req &req, http_res &res);
+
+void on_send_response(void *data);
+
+static constexpr const char* SEND_RESPONSE_MSG = "send_response";
+static constexpr const char* REPLICATION_EVENT_MSG = "replication_event";
\ No newline at end of file
diff --git a/include/collection.h b/include/collection.h
index fc2a5425..099e3c2b 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -142,6 +142,12 @@ public:
uint32_t get_next_seq_id();
+ void set_next_seq_id(uint32_t seq_id);
+
+ void increment_next_seq_id_field();
+
+ static uint32_t deserialize_seq_id_key(std::string serialized_seq_id);
+
uint32_t doc_id_to_seq_id(std::string doc_id);
std::vector<std::string> get_facet_fields();
diff --git a/include/collection_manager.h b/include/collection_manager.h
index b4f53ec2..5bdf4895 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -14,6 +14,8 @@ private:
spp::sparse_hash_map<std::string, Collection*> collections;
+ spp::sparse_hash_map<uint32_t, std::string> collection_id_names;
+
// Auto incrementing ID assigned to each collection
// Using a ID instead of a collection's name makes renaming possible
uint32_t next_collection_id;
@@ -42,6 +44,10 @@ public:
void init(Store *store, const std::string & auth_key);
+ Collection* init_collection(const std::string & collection_meta_json);
+
+ void add_to_collections(Collection* collection);
+
bool auth_key_matches(std::string auth_key_sent);
Collection* create_collection(std::string name, const std::vector<field> & search_fields,
@@ -51,12 +57,16 @@ public:
Collection* get_collection(const std::string & collection_name);
+ Collection* get_collection_with_id(uint32_t collection_id);
+
std::vector<Collection*> get_collections();
Option<bool> drop_collection(std::string collection_name);
uint32_t get_next_collection_id();
+ void set_next_collection_id(uint32_t next_id);
+
Store* get_store();
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
diff --git a/include/http_client.h b/include/http_client.h
new file mode 100644
index 00000000..8c941f7f
--- /dev/null
+++ b/include/http_client.h
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <string>
+#include <curl/curl.h>
+
+/*
+ NOTE: This is a really primitive blocking client meant only for specific Typesense use cases.
+*/
+class HttpClient {
+private:
+ std::string buffer;
+ std::string url;
+public:
+ HttpClient(std::string url): url(url) {
+
+ }
+
+ static size_t curl_write (void *contents, size_t size, size_t nmemb, std::string *s) {
+ s->append((char*)contents, size*nmemb);
+ return size*nmemb;
+ }
+
+ long get_reponse(std::string & response) {
+ CURL *curl = curl_easy_init();
+ curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, HttpClient::curl_write);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
+
+ struct curl_slist *chunk = NULL;
+ chunk = curl_slist_append(chunk, "x-typesense-api-key: abcd");
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);
+
+ curl_easy_perform(curl);
+ long http_code = 0;
+ curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);
+ curl_easy_cleanup(curl);
+ response = buffer;
+ return http_code;
+ }
+};
\ No newline at end of file
diff --git a/include/http_data.h b/include/http_data.h
index cabd0bc0..2e55beb8 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -1,12 +1,21 @@
#pragma once
+
#include <stdint.h>
#include <string>
#include <map>
#include <vector>
+#define H2O_USE_LIBUV 0
+extern "C" {
+ #include "h2o.h"
+}
+
+class HttpServer;
+
struct http_res {
uint32_t status_code;
std::string body;
+ HttpServer* server;
void send_200(const std::string & res_body) {
status_code = 200;
@@ -60,6 +69,7 @@ struct http_res {
};
struct http_req {
+ h2o_req_t* _req;
std::map<std::string, std::string> params;
std::string body;
};
diff --git a/include/http_server.h b/include/http_server.h
index d7de19d7..067d1c95 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -15,6 +15,11 @@ extern "C" {
#include "collection.h"
#include "collection_manager.h"
+struct request_response {
+ http_req* req;
+ http_res* response;
+};
+
class HttpServer {
private:
h2o_globalconf_t config;
@@ -26,6 +31,8 @@ private:
std::vector<route_path> routes;
+ std::map<std::string, void (*)(void*)> message_handlers;
+
const std::string listen_address;
const uint32_t listen_port;
@@ -47,8 +54,6 @@ private:
static int send_401_unauthorized(h2o_req_t *req);
- void send_response(h2o_req_t* req, h2o_generator_t & generator, const http_res & response);
-
static constexpr const char* SEND_RESPONSE_MSG = "send_response";
public:
@@ -64,6 +69,12 @@ public:
void del(const std::string & path, void (*handler)(http_req & req, http_res & res), bool authenticated, bool async = false);
+ void on(const std::string & message, void (*handler)(void*));
+
+ void send_message(const std::string & type, void* data);
+
+ void send_response(http_req* request, const http_res* response);
+
int run();
static constexpr const char* AUTH_HEADER = "x-typesense-api-key";
diff --git a/include/replicator.h b/include/replicator.h
new file mode 100644
index 00000000..bfe3bdc4
--- /dev/null
+++ b/include/replicator.h
@@ -0,0 +1,153 @@
+#pragma once
+
+#include "http_client.h"
+#include "collection_manager.h"
+#include "collection.h"
+#include "string_utils.h"
+#include <json.hpp>
+#include <thread>
+#include <string>
+#include <iostream>
+
+
+struct ReplicationEvent {
+ std::mutex m;
+ std::condition_variable cv;
+ std::string type;
+ uint32_t collection_id;
+ std::string key;
+ std::string value;
+
+ ReplicationEvent(const std::string& type, const uint32_t collection_id,
+ const std::string& key, const std::string& value):
+ type(type), collection_id(collection_id), key(key), value(value) {
+
+ }
+};
+
+class IterateBatchHandler: public rocksdb::WriteBatch::Handler {
+private:
+ HttpServer* server;
+public:
+ IterateBatchHandler(HttpServer* server): server(server) {
+
+ }
+
+ void Put(const rocksdb::Slice& key, const rocksdb::Slice& value) {
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ std::vector<std::string> parts;
+ StringUtils::split(key.ToString(), parts, "_");
+
+ std::string val = value.ToString() + "\n";
+
+ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {
+ // nothing to do here as this is called only when a new collection is created and it's always "0"
+ }
+
+ if(parts.size() == 1 && parts[0] == CollectionManager::NEXT_COLLECTION_ID_KEY) {
+ ReplicationEvent* replication_event = new ReplicationEvent("UPDATE_NEXT_COLLECTION_ID", 0,
+ key.ToString(), value.ToString());
+ server->send_message(REPLICATION_EVENT_MSG, replication_event);
+ }
+
+ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {
+ ReplicationEvent* replication_event = new ReplicationEvent("ADD_COLLECTION_META", 0,
+ key.ToString(), value.ToString());
+ server->send_message(REPLICATION_EVENT_MSG, replication_event);
+ }
+
+ if(parts.size() == 3 && parts[1] == Collection::SEQ_ID_PREFIX) {
+ ReplicationEvent* replication_event = new ReplicationEvent("ADD_DOCUMENT", std::stoi(parts[0]),
+ key.ToString(), value.ToString());
+ server->send_message(REPLICATION_EVENT_MSG, replication_event);
+ }
+ }
+
+ void Delete(const rocksdb::Slice& key) {
+
+ }
+
+ void Merge(const rocksdb::Slice& key, const rocksdb::Slice& value) {
+ std::vector<std::string> parts;
+ StringUtils::split(key.ToString(), parts, "_");
+
+ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {
+ ReplicationEvent* replication_event = new ReplicationEvent("INCR_COLLECTION_NEXT_SEQ", 0,
+ key.ToString(), value.ToString());
+ server->send_message(REPLICATION_EVENT_MSG, replication_event);
+ }
+ }
+};
+
+class Replicator {
+public:
+ static void start(HttpServer* server, const std::string& master_host_port, Store& store) {
+ while(true) {
+ IterateBatchHandler handler(server);
+ uint64_t latest_seq_num = store.get_latest_seq_number();
+ std::cout << "latest_seq_num: " << latest_seq_num << std::endl;
+
+ HttpClient client(master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1));
+
+ std::string json_response;
+ long status_code = client.get_reponse(json_response);
+
+ if(status_code == 200) {
+ nlohmann::json json_content = nlohmann::json::parse(json_response);
+ nlohmann::json updates = json_content["updates"];
+ for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {
+ const std::string update_decoded = StringUtils::base64_decode(*update);
+ rocksdb::WriteBatch write_batch(update_decoded);
+ write_batch.Iterate(&handler);
+ }
+
+ for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {
+ const std::string update_decoded = StringUtils::base64_decode(*update);
+ rocksdb::WriteBatch write_batch(update_decoded);
+ store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);
+ }
+ }
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(3000));
+ }
+ }
+
+ static void on_replication_event(void *data) {
+ ReplicationEvent* replication_event = static_cast<ReplicationEvent*>(data);
+
+ if(replication_event->type == "UPDATE_NEXT_COLLECTION_ID") {
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ collection_manager.set_next_collection_id(std::stoi(replication_event->value));
+ }
+
+ if(replication_event->type == "ADD_COLLECTION_META") {
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ Collection* collection = collection_manager.init_collection(replication_event->value);
+ collection_manager.add_to_collections(collection);
+ }
+
+ if(replication_event->type == "ADD_DOCUMENT") {
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ Collection* collection = collection_manager.get_collection_with_id(replication_event->collection_id);
+ nlohmann::json document = nlohmann::json::parse(replication_event->value);
+
+ // last 4 bytes of the key would be the serialized version of the sequence id
+ std::string serialized_seq_id = replication_event->key.substr(replication_event->key.length() - 4);
+ uint32_t seq_id = Collection::deserialize_seq_id_key(serialized_seq_id);
+ collection->index_in_memory(document, seq_id);
+ }
+
+ if(replication_event->type == "INCR_COLLECTION_NEXT_SEQ") {
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ std::string collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);
+ Collection* collection = collection_manager.get_collection(collection_name);
+ collection->increment_next_seq_id_field();
+ }
+
+ if(replication_event->type == "REMOVE") {
+
+ }
+
+ delete replication_event;
+ }
+};
\ No newline at end of file
diff --git a/include/store.h b/include/store.h
index 482db444..1e1345d5 100644
--- a/include/store.h
+++ b/include/store.h
@@ -40,7 +40,7 @@ enum StoreStatus {
class Store {
private:
- std::string state_dir_path;
+ const std::string state_dir_path;
rocksdb::DB *db;
rocksdb::Options options;
@@ -49,7 +49,7 @@ public:
Store() = delete;
- Store(std::string state_dir_path): state_dir_path(state_dir_path) {
+ Store(const std::string & state_dir_path): state_dir_path(state_dir_path) {
// Optimize RocksDB
options.IncreaseParallelism();
options.OptimizeLevelStyleCompaction();
@@ -80,13 +80,13 @@ public:
return status.ok();
}
- bool contains(const std::string& key) {
+ bool contains(const std::string& key) const {
std::string value;
rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
return status.ok() && !status.IsNotFound();
}
- StoreStatus get(const std::string& key, std::string& value) {
+ StoreStatus get(const std::string& key, std::string& value) const {
rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
if(status.IsNotFound()) {
@@ -129,26 +129,34 @@ public:
db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));
}
- Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number, const uint64_t max_updates) {
- Option<std::vector<std::string>*> updates_op(new std::vector<std::string>());
+ uint64_t get_latest_seq_number() const {
+ return db->GetLatestSequenceNumber();
+ }
+ /*
+ Since `GetLatestSequenceNumber` returns 0 when the DB is empty and when there is 1 record:
+ get_updates_since(0) == get_updates_since(1) - so always query for 1 sequence number greater than the number
+ returned by GetLatestSequenceNumber() locally.
+ */
+ Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number, const uint64_t max_updates) const {
rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;
rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
-
if(!status.ok()) {
- return Option<std::vector<std::string>*>(500, "Could not fetch updates from the disk store.");
+ return Option<std::vector<std::string>*>(204, "Invalid sequence number.");
}
uint64_t num_updates = 0;
+ std::vector<std::string>* updates = new std::vector<std::string>();
+
while(iter->Valid() && num_updates < max_updates) {
rocksdb::BatchResult batch_result = iter->GetBatch();
const std::string & write_batch_serialized = batch_result.writeBatchPtr->Data();
- updates_op.get()->push_back(write_batch_serialized);
+ updates->push_back(write_batch_serialized);
num_updates += 1;
iter->Next();
}
- return updates_op;
+ return Option<std::vector<std::string>*>(updates);
}
void close() {
@@ -157,7 +165,7 @@ public:
}
// Only for internal tests
- rocksdb::DB* _get_db_unsafe() {
+ rocksdb::DB* _get_db_unsafe() const {
return db;
}
diff --git a/src/api.cpp b/src/api.cpp
index cbcb164a..a089f271 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -1,5 +1,6 @@
#include <regex>
#include <chrono>
+#include <thread>
#include <sys/resource.h>
#include "api.h"
#include "string_utils.h"
@@ -335,27 +336,42 @@ void del_remove_document(http_req & req, http_res & res) {
}
void get_replication_updates(http_req & req, http_res & res) {
- if(!StringUtils::is_uint64_t(req.params["seq_number"])) {
- return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");
- }
+ // Could be heavy - spawn a new thread so we don't block the main thread
+ std::thread response_thread([&]() {
+ if(!StringUtils::is_uint64_t(req.params["seq_number"])) {
+ return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");
+ }
- const uint64_t MAX_UPDATES_TO_SEND = 2000;
- uint64_t seq_number = std::stoull(req.params["seq_number"]);
+ const uint64_t MAX_UPDATES_TO_SEND = 2000;
+ uint64_t seq_number = std::stoull(req.params["seq_number"]);
- CollectionManager & collectionManager = CollectionManager::get_instance();
- Store* store = collectionManager.get_store();
- Option<std::vector<std::string>*> updates_op = store->get_updates_since(seq_number, MAX_UPDATES_TO_SEND);
- if(!updates_op.ok()) {
- return res.send(updates_op.code(), updates_op.error());
- }
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Store* store = collectionManager.get_store();
+ Option<std::vector<std::string>*> updates_op = store->get_updates_since(seq_number, MAX_UPDATES_TO_SEND);
+ if(!updates_op.ok()) {
+ res.send(updates_op.code(), updates_op.error());
+ res.server->send_message(SEND_RESPONSE_MSG, new request_response{&req, &res});
+ return ;
+ }
- nlohmann::json json_response;
- json_response["updates"] = nlohmann::json::array();
+ nlohmann::json json_response;
+ json_response["updates"] = nlohmann::json::array();
- std::vector<std::string> *updates = updates_op.get();
- for(const std::string & update: *updates) {
- json_response["updates"].push_back(StringUtils::base64_encode(update));
- }
+ std::vector<std::string> *updates = updates_op.get();
+ for(const std::string & update: *updates) {
+ json_response["updates"].push_back(StringUtils::base64_encode(update));
+ }
- res.send_200(json_response.dump());
+ res.send_200(json_response.dump());
+ res.server->send_message(SEND_RESPONSE_MSG, new request_response{&req, &res});
+ delete updates;
+ });
+
+ response_thread.detach();
+}
+
+void on_send_response(void *data) {
+ request_response* req_res = static_cast<request_response*>(data);
+ req_res->response->server->send_response(req_res->req, req_res->response);
+ delete req_res;
}
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index ecfcb45e..b29c710a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -53,6 +53,14 @@ uint32_t Collection::get_next_seq_id() {
return next_seq_id++;
}
+void Collection::set_next_seq_id(uint32_t seq_id) {
+ next_seq_id = seq_id;
+}
+
+void Collection::increment_next_seq_id_field() {
+ next_seq_id++;
+}
+
Option<std::string> Collection::add(const std::string & json_str) {
nlohmann::json document = nlohmann::json::parse(json_str);
@@ -73,8 +81,8 @@ Option<std::string> Collection::add(const std::string & json_str) {
return Option<std::string>(index_memory_op.code(), index_memory_op.error());
}
- store->insert(get_seq_id_key(seq_id), document.dump());
store->insert(get_doc_id_key(document["id"]), seq_id_str);
+ store->insert(get_seq_id_key(seq_id), document.dump());
return Option<std::string>(doc_id);
}
@@ -1384,7 +1392,8 @@ std::string Collection::get_next_seq_id_key(const std::string & collection_name)
}
std::string Collection::get_seq_id_key(uint32_t seq_id) {
- // We can't simply do std::to_string() because we want to preserve the byte order
+ // We can't simply do std::to_string() because we want to preserve the byte order.
+ // & 0xFF masks all but the lowest eight bits.
unsigned char bytes[4];
bytes[0] = (unsigned char) ((seq_id >> 24) & 0xFF);
bytes[1] = (unsigned char) ((seq_id >> 16) & 0xFF);
@@ -1394,6 +1403,12 @@ std::string Collection::get_seq_id_key(uint32_t seq_id) {
return get_seq_id_collection_prefix() + "_" + std::string(bytes, bytes+4);
}
+uint32_t Collection::deserialize_seq_id_key(std::string serialized_seq_id) {
+ uint32_t seq_id = ((serialized_seq_id[0] & 0xFF) << 24) | ((serialized_seq_id[1] & 0xFF) << 16) |
+ ((serialized_seq_id[2] & 0xFF) << 8) | (serialized_seq_id[3] & 0xFF);
+ return seq_id;
+}
+
std::string Collection::get_doc_id_key(const std::string & doc_id) {
return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;
}
@@ -1440,7 +1455,7 @@ spp::sparse_hash_map<std::string, field> Collection::get_schema() {
};
std::string Collection::get_meta_key(const std::string & collection_name) {
- return COLLECTION_META_PREFIX + collection_name;
+ return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;
}
std::string Collection::get_seq_id_collection_prefix() {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 9371210c..1701221c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -8,6 +8,55 @@ CollectionManager::CollectionManager() {
}
+Collection* CollectionManager::init_collection(const std::string & collection_meta_json) {
+ nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);
+ std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();
+
+ std::vector<field> search_fields;
+ nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
+
+ for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {
+ search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
+ }
+
+ std::vector<field> facet_fields;
+ nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];
+
+ for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {
+ facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
+ }
+
+ std::string collection_next_seq_id_str;
+ store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);
+ uint32_t collection_next_seq_id = collection_next_seq_id_str.size() == 0 ? 0 :
+ (const uint32_t) std::stoi(collection_next_seq_id_str);
+
+ std::vector<field> collection_sort_fields;
+ nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];
+
+ for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {
+ collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
+ }
+
+ std::string token_ranking_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();
+
+ Collection* collection = new Collection(this_collection_name,
+ collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
+ collection_next_seq_id,
+ store,
+ search_fields,
+ facet_fields,
+ collection_sort_fields,
+ token_ranking_field);
+
+ return collection;
+}
+
+void CollectionManager::add_to_collections(Collection* collection) {
+ collections.emplace(Collection::get_meta_key(collection->get_name()), collection);
+ collection_id_names.emplace(collection->get_collection_id(), collection->get_name());
+}
+
void CollectionManager::init(Store *store, const std::string & auth_key) {
this->store = store;
this->auth_key = auth_key;
@@ -18,52 +67,13 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {
next_collection_id = (uint32_t) stoi(next_collection_id_str);
} else {
next_collection_id = 0;
- store->insert(NEXT_COLLECTION_ID_KEY, std::to_string(next_collection_id));
}
std::vector<std::string> collection_meta_jsons;
store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);
for(auto & collection_meta_json: collection_meta_jsons) {
- nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);
- std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();
-
- std::vector<field> search_fields;
- nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
-
- for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {
- search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
- }
-
- std::vector<field> facet_fields;
- nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];
-
- for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {
- facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
- }
-
- std::string collection_next_seq_id_str;
- store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);
-
- uint32_t collection_next_seq_id = (const uint32_t) std::stoi(collection_next_seq_id_str);
-
- std::vector<field> collection_sort_fields;
- nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];
-
- for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {
- collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
- }
-
- std::string token_ranking_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();
-
- Collection* collection = new Collection(this_collection_name,
- collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
- collection_next_seq_id,
- store,
- search_fields,
- facet_fields,
- collection_sort_fields,
- token_ranking_field);
+ Collection* collection = init_collection(collection_meta_json);
// Fetch records from the store and re-create memory index
std::vector<std::string> documents;
@@ -80,7 +90,7 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {
delete iter;
- collections.emplace(Collection::get_meta_key(this_collection_name), collection);
+ add_to_collections(collection);
}
std::cout << "Finished restoring all collections from disk." << std::endl;
@@ -133,14 +143,13 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, facet_fields,
sort_fields, token_ranking_field);
+ next_collection_id++;
- store->insert(Collection::get_meta_key(name), collection_meta.dump());
store->insert(Collection::get_next_seq_id_key(name), std::to_string(0));
-
- next_collection_id++;
+ store->insert(Collection::get_meta_key(name), collection_meta.dump());
store->insert(NEXT_COLLECTION_ID_KEY, std::to_string(next_collection_id));
- collections.emplace(Collection::get_meta_key(name), new_collection);
+ add_to_collections(new_collection);
return new_collection;
}
@@ -153,6 +162,14 @@ Collection* CollectionManager::get_collection(const std::string & collection_nam
return nullptr;
}
+Collection* CollectionManager::get_collection_with_id(uint32_t collection_id) {
+ if(collection_id_names.count(collection_id) != 0) {
+ return get_collection(collection_id_names.at(collection_id));
+ }
+
+ return nullptr;
+}
+
std::vector<Collection*> CollectionManager::get_collections() {
std::vector<Collection*> collection_vec;
for(auto kv: collections) {
@@ -186,6 +203,7 @@ Option<bool> CollectionManager::drop_collection(std::string collection_name) {
delete iter;
collections.erase(Collection::get_meta_key(collection_name));
+ collection_id_names.erase(collection->get_collection_id());
delete collection;
collection = nullptr;
@@ -197,6 +215,10 @@ uint32_t CollectionManager::get_next_collection_id() {
return next_collection_id;
}
+void CollectionManager::set_next_collection_id(uint32_t next_id) {
+ next_collection_id = next_id;
+}
+
Store* CollectionManager::get_store() {
return store;
}
\ No newline at end of file
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 880cb9e3..18ac23ca 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -11,11 +11,6 @@ struct h2o_custom_req_handler_t {
HttpServer* http_server;
};
-struct request_response {
- h2o_req_t* req;
- http_res* response;
-};
-
struct h2o_custom_res_message_t {
h2o_multithread_message_t super;
HttpServer* http_server;
@@ -98,11 +93,9 @@ void HttpServer::on_message(h2o_multithread_receiver_t *receiver, h2o_linklist_t
h2o_multithread_message_t *message = H2O_STRUCT_FROM_MEMBER(h2o_multithread_message_t, link, messages->next);
h2o_custom_res_message_t *custom_message = reinterpret_cast<h2o_custom_res_message_t*>(message);
- if(custom_message->type == SEND_RESPONSE_MSG) {
- request_response* req_res = static_cast<request_response*>(custom_message->data);
- custom_message->http_server->send_response(req_res->req, generator, *req_res->response);
- delete req_res->response;
- delete req_res;
+ if(custom_message->http_server->message_handlers.count(custom_message->type) != 0) {
+ auto handler = custom_message->http_server->message_handlers.at(custom_message->type);
+ (handler)(custom_message->data);
}
h2o_linklist_unlink(&message->link);
@@ -224,30 +217,17 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
}
}
- if(rpath.async) {
- /*
- Must call h2o_start_response and h2o_send functions within the same thread that
- called the on_req callback.
- */
- std::thread response_thread([=]() {
- http_req request = {query_map, req_body};
- http_res* response = new http_res();
- (rpath.handler)(request, *response);
- request_response* req_res = new request_response{req, response};
- h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{NULL}}, self->http_server,
- SEND_RESPONSE_MSG, req_res};
- h2o_multithread_send_message(self->http_server->message_receiver, &message->super);
- });
-
- response_thread.detach();
- return 0;
+ http_req* request = new http_req{req, query_map, req_body};
+ http_res* response = new http_res();
+ response->server = self->http_server;
+ (rpath.handler)(*request, *response);
+
+ if(!rpath.async) {
+ // If a handler is marked async, it's assumed that it's responsible for sending the response itself
+ // later in an async fashion by calling into the main http thread via a message
+ self->http_server->send_response(request, response);
}
- http_req request = {query_map, req_body};
- http_res response;
- h2o_generator_t generator = {NULL, NULL};
- (rpath.handler)(request, response);
- self->http_server->send_response(req, generator, response);
return 0;
}
}
@@ -263,13 +243,24 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
return 0;
}
-void HttpServer::send_response(h2o_req_t* req, h2o_generator_t & generator, const http_res & response) {
- h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);
- req->res.status = response.status_code;
- req->res.reason = get_status_reason(response.status_code);
+void HttpServer::send_message(const std::string & type, void* data) {
+ h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{NULL}}, this, type.c_str(), data};
+ h2o_multithread_send_message(message_receiver, &message->super);
+}
+
+void HttpServer::send_response(http_req* request, const http_res* response) {
+ h2o_req_t* req = request->_req;
+ h2o_generator_t generator = {NULL, NULL};
+
+ h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
+ req->res.status = response->status_code;
+ req->res.reason = get_status_reason(response->status_code);
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL, H2O_STRLIT("application/json; charset=utf-8"));
h2o_start_response(req, &generator);
h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
+
+ delete request;
+ delete response;
}
int HttpServer::send_401_unauthorized(h2o_req_t *req) {
@@ -312,6 +303,11 @@ void HttpServer::del(const std::string & path, void (*handler)(http_req &, http_
routes.push_back(rpath);
}
+
+void HttpServer::on(const std::string & message, void (*handler)(void*)) {
+ message_handlers.emplace(message, handler);
+}
+
HttpServer::~HttpServer() {
delete accept_ctx;
delete message_receiver;
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 7f67f891..71f47e3a 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -1,8 +1,13 @@
#include <signal.h>
#include <iostream>
+#include <vector>
+#include <string>
+#include <thread>
#include <cmdline.h>
#include "http_server.h"
#include "api.h"
+#include "string_utils.h"
+#include "replicator.h"
void catch_interrupt(int sig) {
std::cout << "Quitting Typesense..." << std::endl;
@@ -14,6 +19,8 @@ int main(int argc, char **argv) {
options.add<std::string>("api-auth-key", 'k', "Key for authenticating the API endpoints.", true);
options.add<std::string>("listen-address", 'a', "Address to which Typesense server binds.", false, "0.0.0.0");
options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8080);
+ options.add<std::string>("master", 'm', "<master_address>:<master_port> combination to start the server as a "
+ "read-only replica.", false, "");
options.parse_check(argc, argv);
signal(SIGINT, catch_interrupt);
@@ -42,6 +49,27 @@ int main(int argc, char **argv) {
// replication
server.get("/replication/updates", get_replication_updates, true, true);
+ server.on(SEND_RESPONSE_MSG, on_send_response);
+ server.on(REPLICATION_EVENT_MSG, Replicator::on_replication_event);
+
+ // start a background replication thread if the server is started as a read-only replica
+ if(!options.get<std::string>("master").empty()) {
+ const std::string & master_host_port = options.get<std::string>("master");
+ std::vector<std::string> parts;
+ StringUtils::split(master_host_port, parts, ":");
+ if(parts.size() != 2) {
+ std::cerr << "Invalid value for --master option. Usage: <master_address>:<master_port>" << std::endl;
+ return 1;
+ }
+
+ std::cout << "Typesense server started as a read-only replica... Spawning replication thread..." << std::endl;
+ std::thread replication_thread([&server, &master_host_port, &store]() {
+ Replicator::start(&server, master_host_port, store);
+ });
+
+ replication_thread.detach();
+ }
+
server.run();
return 0;
}
\ No newline at end of file