Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Basics of background replication.
Kishore Nallan58a10877ee23
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 6094b8fe..0dea3d4f 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -90,7 +90,7 @@ if(NOT APPLE)list(APPEND ROCKSDB_LIBS rt)endif()-target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})-target_link_libraries(benchmark for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(typesense-server h2o-evloop for pthread curl ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(search for pthread h2o-evloop curl ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})+target_link_libraries(benchmark for pthread curl h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})diff --git a/TODO.md b/TODO.mdindex 57361200..92f53ef3 100644--- a/TODO.md+++ b/TODO.md@@ -57,6 +57,8 @@- ~~only last token should be prefix searched~~- ~~Prefix-search strings should not be null terminated~~- ~~sort results by float field~~+- Collection Manager collections map should store plain collection name+- init_collection of Collection manager should probably take seq_id as param- Validate before string to int conversion in the http api layer- When field of "id" but not string, what happens?- Typo in prefix searchdiff --git a/docker/development.Dockerfile b/docker/development.Dockerfileindex 36a3af6d..54f1e4d8 100644--- a/docker/development.Dockerfile+++ b/docker/development.Dockerfile@@ -24,5 +24,7 @@ RUN make -C /opt/openssl-1.0.2k dependRUN make -C /opt/openssl-1.0.2k -j4RUN make -C /opt/openssl-1.0.2k install+RUN apt-get install curl+ENV CC /usr/local/gcc-4.9.2/bin/gccENV CXX /usr/local/gcc-4.9.2/bin/g++diff --git a/include/api.h b/include/api.hindex 1a88181b..f5147edc 100644--- a/include/api.h+++ b/include/api.h@@ -18,4 +18,9 @@ void get_fetch_document(http_req & req, http_res & res);void del_remove_document(http_req & req, http_res & res);-void get_replication_updates(http_req &req, http_res &res);\ No newline at end of file+void get_replication_updates(http_req &req, http_res &res);++void on_send_response(void *data);++static constexpr const char* SEND_RESPONSE_MSG = "send_response";+static constexpr const char* REPLICATION_EVENT_MSG = "replication_event";\ No newline at end of filediff --git a/include/collection.h b/include/collection.hindex fc2a5425..099e3c2b 100644--- a/include/collection.h+++ b/include/collection.h@@ -142,6 +142,12 @@ public:uint32_t get_next_seq_id();+ void set_next_seq_id(uint32_t seq_id);++ void increment_next_seq_id_field();++ static uint32_t deserialize_seq_id_key(std::string serialized_seq_id);+uint32_t doc_id_to_seq_id(std::string doc_id);std::vector<std::string> get_facet_fields();diff --git a/include/collection_manager.h b/include/collection_manager.hindex b4f53ec2..5bdf4895 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -14,6 +14,8 @@ private:spp::sparse_hash_map<std::string, Collection*> collections;+ spp::sparse_hash_map<uint32_t, std::string> collection_id_names;+// Auto incrementing ID assigned to each collection// Using a ID instead of a collection's name makes renaming possibleuint32_t next_collection_id;@@ -42,6 +44,10 @@ public:void init(Store *store, const std::string & auth_key);+ Collection* init_collection(const std::string & collection_meta_json);++ void add_to_collections(Collection* collection);+bool auth_key_matches(std::string auth_key_sent);Collection* create_collection(std::string name, const std::vector<field> & search_fields,@@ -51,12 +57,16 @@ public:Collection* get_collection(const std::string & collection_name);+ Collection* get_collection_with_id(uint32_t collection_id);+std::vector<Collection*> get_collections();Option<bool> drop_collection(std::string collection_name);uint32_t get_next_collection_id();+ void set_next_collection_id(uint32_t next_id);+Store* get_store();static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";diff --git a/include/http_client.h b/include/http_client.hnew file mode 100644index 00000000..8c941f7f--- /dev/null+++ b/include/http_client.h@@ -0,0 +1,40 @@+#pragma once++#include <string>+#include <curl/curl.h>++/*+ NOTE: This is a really primitive blocking client meant only for specific Typesense use cases.+*/+class HttpClient {+private:+ std::string buffer;+ std::string url;+public:+ HttpClient(std::string url): url(url) {++ }++ static size_t curl_write (void *contents, size_t size, size_t nmemb, std::string *s) {+ s->append((char*)contents, size*nmemb);+ return size*nmemb;+ }++ long get_reponse(std::string & response) {+ CURL *curl = curl_easy_init();+ curl_easy_setopt(curl, CURLOPT_URL, url.c_str());+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, HttpClient::curl_write);+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);++ struct curl_slist *chunk = NULL;+ chunk = curl_slist_append(chunk, "x-typesense-api-key: abcd");+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk);++ curl_easy_perform(curl);+ long http_code = 0;+ curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);+ curl_easy_cleanup(curl);+ response = buffer;+ return http_code;+ }+};\ No newline at end of filediff --git a/include/http_data.h b/include/http_data.hindex cabd0bc0..2e55beb8 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -1,12 +1,21 @@#pragma once+#include <stdint.h>#include <string>#include <map>#include <vector>+#define H2O_USE_LIBUV 0+extern "C" {+ #include "h2o.h"+}++class HttpServer;+struct http_res {uint32_t status_code;std::string body;+ HttpServer* server;void send_200(const std::string & res_body) {status_code = 200;@@ -60,6 +69,7 @@ struct http_res {};struct http_req {+ h2o_req_t* _req;std::map<std::string, std::string> params;std::string body;};diff --git a/include/http_server.h b/include/http_server.hindex d7de19d7..067d1c95 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -15,6 +15,11 @@ extern "C" {#include "collection.h"#include "collection_manager.h"+struct request_response {+ http_req* req;+ http_res* response;+};+class HttpServer {private:h2o_globalconf_t config;@@ -26,6 +31,8 @@ private:std::vector<route_path> routes;+ std::map<std::string, void (*)(void*)> message_handlers;+const std::string listen_address;const uint32_t listen_port;@@ -47,8 +54,6 @@ private:static int send_401_unauthorized(h2o_req_t *req);- void send_response(h2o_req_t* req, h2o_generator_t & generator, const http_res & response);-static constexpr const char* SEND_RESPONSE_MSG = "send_response";public:@@ -64,6 +69,12 @@ public:void del(const std::string & path, void (*handler)(http_req & req, http_res & res), bool authenticated, bool async = false);+ void on(const std::string & message, void (*handler)(void*));++ void send_message(const std::string & type, void* data);++ void send_response(http_req* request, const http_res* response);+int run();static constexpr const char* AUTH_HEADER = "x-typesense-api-key";diff --git a/include/replicator.h b/include/replicator.hnew file mode 100644index 00000000..bfe3bdc4--- /dev/null+++ b/include/replicator.h@@ -0,0 +1,153 @@+#pragma once++#include "http_client.h"+#include "collection_manager.h"+#include "collection.h"+#include "string_utils.h"+#include <json.hpp>+#include <thread>+#include <string>+#include <iostream>+++struct ReplicationEvent {+ std::mutex m;+ std::condition_variable cv;+ std::string type;+ uint32_t collection_id;+ std::string key;+ std::string value;++ ReplicationEvent(const std::string& type, const uint32_t collection_id,+ const std::string& key, const std::string& value):+ type(type), collection_id(collection_id), key(key), value(value) {++ }+};++class IterateBatchHandler: public rocksdb::WriteBatch::Handler {+private:+ HttpServer* server;+public:+ IterateBatchHandler(HttpServer* server): server(server) {++ }++ void Put(const rocksdb::Slice& key, const rocksdb::Slice& value) {+ CollectionManager & collection_manager = CollectionManager::get_instance();+ std::vector<std::string> parts;+ StringUtils::split(key.ToString(), parts, "_");++ std::string val = value.ToString() + "\n";++ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {+ // nothing to do here as this is called only when a new collection is created and it's always "0"+ }++ if(parts.size() == 1 && parts[0] == CollectionManager::NEXT_COLLECTION_ID_KEY) {+ ReplicationEvent* replication_event = new ReplicationEvent("UPDATE_NEXT_COLLECTION_ID", 0,+ key.ToString(), value.ToString());+ server->send_message(REPLICATION_EVENT_MSG, replication_event);+ }++ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {+ ReplicationEvent* replication_event = new ReplicationEvent("ADD_COLLECTION_META", 0,+ key.ToString(), value.ToString());+ server->send_message(REPLICATION_EVENT_MSG, replication_event);+ }++ if(parts.size() == 3 && parts[1] == Collection::SEQ_ID_PREFIX) {+ ReplicationEvent* replication_event = new ReplicationEvent("ADD_DOCUMENT", std::stoi(parts[0]),+ key.ToString(), value.ToString());+ server->send_message(REPLICATION_EVENT_MSG, replication_event);+ }+ }++ void Delete(const rocksdb::Slice& key) {++ }++ void Merge(const rocksdb::Slice& key, const rocksdb::Slice& value) {+ std::vector<std::string> parts;+ StringUtils::split(key.ToString(), parts, "_");++ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_NEXT_SEQ_PREFIX) {+ ReplicationEvent* replication_event = new ReplicationEvent("INCR_COLLECTION_NEXT_SEQ", 0,+ key.ToString(), value.ToString());+ server->send_message(REPLICATION_EVENT_MSG, replication_event);+ }+ }+};++class Replicator {+public:+ static void start(HttpServer* server, const std::string& master_host_port, Store& store) {+ while(true) {+ IterateBatchHandler handler(server);+ uint64_t latest_seq_num = store.get_latest_seq_number();+ std::cout << "latest_seq_num: " << latest_seq_num << std::endl;++ HttpClient client(master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1));++ std::string json_response;+ long status_code = client.get_reponse(json_response);++ if(status_code == 200) {+ nlohmann::json json_content = nlohmann::json::parse(json_response);+ nlohmann::json updates = json_content["updates"];+ for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {+ const std::string update_decoded = StringUtils::base64_decode(*update);+ rocksdb::WriteBatch write_batch(update_decoded);+ write_batch.Iterate(&handler);+ }++ for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {+ const std::string update_decoded = StringUtils::base64_decode(*update);+ rocksdb::WriteBatch write_batch(update_decoded);+ store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);+ }+ }++ std::this_thread::sleep_for(std::chrono::milliseconds(3000));+ }+ }++ static void on_replication_event(void *data) {+ ReplicationEvent* replication_event = static_cast<ReplicationEvent*>(data);++ if(replication_event->type == "UPDATE_NEXT_COLLECTION_ID") {+ CollectionManager & collection_manager = CollectionManager::get_instance();+ collection_manager.set_next_collection_id(std::stoi(replication_event->value));+ }++ if(replication_event->type == "ADD_COLLECTION_META") {+ CollectionManager & collection_manager = CollectionManager::get_instance();+ Collection* collection = collection_manager.init_collection(replication_event->value);+ collection_manager.add_to_collections(collection);+ }++ if(replication_event->type == "ADD_DOCUMENT") {+ CollectionManager & collection_manager = CollectionManager::get_instance();+ Collection* collection = collection_manager.get_collection_with_id(replication_event->collection_id);+ nlohmann::json document = nlohmann::json::parse(replication_event->value);++ // last 4 bytes of the key would be the serialized version of the sequence id+ std::string serialized_seq_id = replication_event->key.substr(replication_event->key.length() - 4);+ uint32_t seq_id = Collection::deserialize_seq_id_key(serialized_seq_id);+ collection->index_in_memory(document, seq_id);+ }++ if(replication_event->type == "INCR_COLLECTION_NEXT_SEQ") {+ CollectionManager & collection_manager = CollectionManager::get_instance();+ std::string collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);+ Collection* collection = collection_manager.get_collection(collection_name);+ collection->increment_next_seq_id_field();+ }++ if(replication_event->type == "REMOVE") {++ }++ delete replication_event;+ }+};\ No newline at end of filediff --git a/include/store.h b/include/store.hindex 482db444..1e1345d5 100644--- a/include/store.h+++ b/include/store.h@@ -40,7 +40,7 @@ enum StoreStatus {class Store {private:- std::string state_dir_path;+ const std::string state_dir_path;rocksdb::DB *db;rocksdb::Options options;@@ -49,7 +49,7 @@ public:Store() = delete;- Store(std::string state_dir_path): state_dir_path(state_dir_path) {+ Store(const std::string & state_dir_path): state_dir_path(state_dir_path) {// Optimize RocksDBoptions.IncreaseParallelism();options.OptimizeLevelStyleCompaction();@@ -80,13 +80,13 @@ public:return status.ok();}- bool contains(const std::string& key) {+ bool contains(const std::string& key) const {std::string value;rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);return status.ok() && !status.IsNotFound();}- StoreStatus get(const std::string& key, std::string& value) {+ StoreStatus get(const std::string& key, std::string& value) const {rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);if(status.IsNotFound()) {@@ -129,26 +129,34 @@ public:db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));}- Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number, const uint64_t max_updates) {- Option<std::vector<std::string>*> updates_op(new std::vector<std::string>());+ uint64_t get_latest_seq_number() const {+ return db->GetLatestSequenceNumber();+ }+ /*+ Since `GetLatestSequenceNumber` returns 0 when the DB is empty and when there is 1 record:+ get_updates_since(0) == get_updates_since(1) - so always query for 1 sequence number greater than the number+ returned by GetLatestSequenceNumber() locally.+ */+ Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number, const uint64_t max_updates) const {rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);-if(!status.ok()) {- return Option<std::vector<std::string>*>(500, "Could not fetch updates from the disk store.");+ return Option<std::vector<std::string>*>(204, "Invalid sequence number.");}uint64_t num_updates = 0;+ std::vector<std::string>* updates = new std::vector<std::string>();+while(iter->Valid() && num_updates < max_updates) {rocksdb::BatchResult batch_result = iter->GetBatch();const std::string & write_batch_serialized = batch_result.writeBatchPtr->Data();- updates_op.get()->push_back(write_batch_serialized);+ updates->push_back(write_batch_serialized);num_updates += 1;iter->Next();}- return updates_op;+ return Option<std::vector<std::string>*>(updates);}void close() {@@ -157,7 +165,7 @@ public:}// Only for internal tests- rocksdb::DB* _get_db_unsafe() {+ rocksdb::DB* _get_db_unsafe() const {return db;}diff --git a/src/api.cpp b/src/api.cppindex cbcb164a..a089f271 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -1,5 +1,6 @@#include <regex>#include <chrono>+#include <thread>#include <sys/resource.h>#include "api.h"#include "string_utils.h"@@ -335,27 +336,42 @@ void del_remove_document(http_req & req, http_res & res) {}void get_replication_updates(http_req & req, http_res & res) {- if(!StringUtils::is_uint64_t(req.params["seq_number"])) {- return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");- }+ // Could be heavy - spawn a new thread so we don't block the main thread+ std::thread response_thread([&]() {+ if(!StringUtils::is_uint64_t(req.params["seq_number"])) {+ return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");+ }- const uint64_t MAX_UPDATES_TO_SEND = 2000;- uint64_t seq_number = std::stoull(req.params["seq_number"]);+ const uint64_t MAX_UPDATES_TO_SEND = 2000;+ uint64_t seq_number = std::stoull(req.params["seq_number"]);- CollectionManager & collectionManager = CollectionManager::get_instance();- Store* store = collectionManager.get_store();- Option<std::vector<std::string>*> updates_op = store->get_updates_since(seq_number, MAX_UPDATES_TO_SEND);- if(!updates_op.ok()) {- return res.send(updates_op.code(), updates_op.error());- }+ CollectionManager & collectionManager = CollectionManager::get_instance();+ Store* store = collectionManager.get_store();+ Option<std::vector<std::string>*> updates_op = store->get_updates_since(seq_number, MAX_UPDATES_TO_SEND);+ if(!updates_op.ok()) {+ res.send(updates_op.code(), updates_op.error());+ res.server->send_message(SEND_RESPONSE_MSG, new request_response{&req, &res});+ return ;+ }- nlohmann::json json_response;- json_response["updates"] = nlohmann::json::array();+ nlohmann::json json_response;+ json_response["updates"] = nlohmann::json::array();- std::vector<std::string> *updates = updates_op.get();- for(const std::string & update: *updates) {- json_response["updates"].push_back(StringUtils::base64_encode(update));- }+ std::vector<std::string> *updates = updates_op.get();+ for(const std::string & update: *updates) {+ json_response["updates"].push_back(StringUtils::base64_encode(update));+ }- res.send_200(json_response.dump());+ res.send_200(json_response.dump());+ res.server->send_message(SEND_RESPONSE_MSG, new request_response{&req, &res});+ delete updates;+ });++ response_thread.detach();+}++void on_send_response(void *data) {+ request_response* req_res = static_cast<request_response*>(data);+ req_res->response->server->send_response(req_res->req, req_res->response);+ delete req_res;}\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex ecfcb45e..b29c710a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -53,6 +53,14 @@ uint32_t Collection::get_next_seq_id() {return next_seq_id++;}+void Collection::set_next_seq_id(uint32_t seq_id) {+ next_seq_id = seq_id;+}++void Collection::increment_next_seq_id_field() {+ next_seq_id++;+}+Option<std::string> Collection::add(const std::string & json_str) {nlohmann::json document = nlohmann::json::parse(json_str);@@ -73,8 +81,8 @@ Option<std::string> Collection::add(const std::string & json_str) {return Option<std::string>(index_memory_op.code(), index_memory_op.error());}- store->insert(get_seq_id_key(seq_id), document.dump());store->insert(get_doc_id_key(document["id"]), seq_id_str);+ store->insert(get_seq_id_key(seq_id), document.dump());return Option<std::string>(doc_id);}@@ -1384,7 +1392,8 @@ std::string Collection::get_next_seq_id_key(const std::string & collection_name)}std::string Collection::get_seq_id_key(uint32_t seq_id) {- // We can't simply do std::to_string() because we want to preserve the byte order+ // We can't simply do std::to_string() because we want to preserve the byte order.+ // & 0xFF masks all but the lowest eight bits.unsigned char bytes[4];bytes[0] = (unsigned char) ((seq_id >> 24) & 0xFF);bytes[1] = (unsigned char) ((seq_id >> 16) & 0xFF);@@ -1394,6 +1403,12 @@ std::string Collection::get_seq_id_key(uint32_t seq_id) {return get_seq_id_collection_prefix() + "_" + std::string(bytes, bytes+4);}+uint32_t Collection::deserialize_seq_id_key(std::string serialized_seq_id) {+ uint32_t seq_id = ((serialized_seq_id[0] & 0xFF) << 24) | ((serialized_seq_id[1] & 0xFF) << 16) |+ ((serialized_seq_id[2] & 0xFF) << 8) | (serialized_seq_id[3] & 0xFF);+ return seq_id;+}+std::string Collection::get_doc_id_key(const std::string & doc_id) {return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;}@@ -1440,7 +1455,7 @@ spp::sparse_hash_map<std::string, field> Collection::get_schema() {};std::string Collection::get_meta_key(const std::string & collection_name) {- return COLLECTION_META_PREFIX + collection_name;+ return std::string(COLLECTION_META_PREFIX) + "_" + collection_name;}std::string Collection::get_seq_id_collection_prefix() {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 9371210c..1701221c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -8,6 +8,55 @@ CollectionManager::CollectionManager() {}+Collection* CollectionManager::init_collection(const std::string & collection_meta_json) {+ nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);+ std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();++ std::vector<field> search_fields;+ nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];++ for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {+ search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});+ }++ std::vector<field> facet_fields;+ nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];++ for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {+ facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});+ }++ std::string collection_next_seq_id_str;+ store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);+ uint32_t collection_next_seq_id = collection_next_seq_id_str.size() == 0 ? 0 :+ (const uint32_t) std::stoi(collection_next_seq_id_str);++ std::vector<field> collection_sort_fields;+ nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];++ for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {+ collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});+ }++ std::string token_ranking_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();++ Collection* collection = new Collection(this_collection_name,+ collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),+ collection_next_seq_id,+ store,+ search_fields,+ facet_fields,+ collection_sort_fields,+ token_ranking_field);++ return collection;+}++void CollectionManager::add_to_collections(Collection* collection) {+ collections.emplace(Collection::get_meta_key(collection->get_name()), collection);+ collection_id_names.emplace(collection->get_collection_id(), collection->get_name());+}+void CollectionManager::init(Store *store, const std::string & auth_key) {this->store = store;this->auth_key = auth_key;@@ -18,52 +67,13 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {next_collection_id = (uint32_t) stoi(next_collection_id_str);} else {next_collection_id = 0;- store->insert(NEXT_COLLECTION_ID_KEY, std::to_string(next_collection_id));}std::vector<std::string> collection_meta_jsons;store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);for(auto & collection_meta_json: collection_meta_jsons) {- nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);- std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();-- std::vector<field> search_fields;- nlohmann::json search_fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];-- for (nlohmann::json::iterator it = search_fields_map.begin(); it != search_fields_map.end(); ++it) {- search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});- }-- std::vector<field> facet_fields;- nlohmann::json facet_fields_map = collection_meta[COLLECTION_FACET_FIELDS_KEY];-- for (nlohmann::json::iterator it = facet_fields_map.begin(); it != facet_fields_map.end(); ++it) {- facet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});- }-- std::string collection_next_seq_id_str;- store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);-- uint32_t collection_next_seq_id = (const uint32_t) std::stoi(collection_next_seq_id_str);-- std::vector<field> collection_sort_fields;- nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];-- for (nlohmann::json::iterator it = sort_fields_map.begin(); it != sort_fields_map.end(); ++it) {- collection_sort_fields.push_back({it.value()[fields::name], it.value()[fields::type]});- }-- std::string token_ranking_field = collection_meta[COLLECTION_TOKEN_ORDERING_FIELD_KEY].get<std::string>();-- Collection* collection = new Collection(this_collection_name,- collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),- collection_next_seq_id,- store,- search_fields,- facet_fields,- collection_sort_fields,- token_ranking_field);+ Collection* collection = init_collection(collection_meta_json);// Fetch records from the store and re-create memory indexstd::vector<std::string> documents;@@ -80,7 +90,7 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {delete iter;- collections.emplace(Collection::get_meta_key(this_collection_name), collection);+ add_to_collections(collection);}std::cout << "Finished restoring all collections from disk." << std::endl;@@ -133,14 +143,13 @@ Collection* CollectionManager::create_collection(std::string name, const std::veCollection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, facet_fields,sort_fields, token_ranking_field);+ next_collection_id++;- store->insert(Collection::get_meta_key(name), collection_meta.dump());store->insert(Collection::get_next_seq_id_key(name), std::to_string(0));-- next_collection_id++;+ store->insert(Collection::get_meta_key(name), collection_meta.dump());store->insert(NEXT_COLLECTION_ID_KEY, std::to_string(next_collection_id));- collections.emplace(Collection::get_meta_key(name), new_collection);+ add_to_collections(new_collection);return new_collection;}@@ -153,6 +162,14 @@ Collection* CollectionManager::get_collection(const std::string & collection_namreturn nullptr;}+Collection* CollectionManager::get_collection_with_id(uint32_t collection_id) {+ if(collection_id_names.count(collection_id) != 0) {+ return get_collection(collection_id_names.at(collection_id));+ }++ return nullptr;+}+std::vector<Collection*> CollectionManager::get_collections() {std::vector<Collection*> collection_vec;for(auto kv: collections) {@@ -186,6 +203,7 @@ Option<bool> CollectionManager::drop_collection(std::string collection_name) {delete iter;collections.erase(Collection::get_meta_key(collection_name));+ collection_id_names.erase(collection->get_collection_id());delete collection;collection = nullptr;@@ -197,6 +215,10 @@ uint32_t CollectionManager::get_next_collection_id() {return next_collection_id;}+void CollectionManager::set_next_collection_id(uint32_t next_id) {+ next_collection_id = next_id;+}+Store* CollectionManager::get_store() {return store;}\ No newline at end of filediff --git a/src/http_server.cpp b/src/http_server.cppindex 880cb9e3..18ac23ca 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -11,11 +11,6 @@ struct h2o_custom_req_handler_t {HttpServer* http_server;};-struct request_response {- h2o_req_t* req;- http_res* response;-};-struct h2o_custom_res_message_t {h2o_multithread_message_t super;HttpServer* http_server;@@ -98,11 +93,9 @@ void HttpServer::on_message(h2o_multithread_receiver_t *receiver, h2o_linklist_th2o_multithread_message_t *message = H2O_STRUCT_FROM_MEMBER(h2o_multithread_message_t, link, messages->next);h2o_custom_res_message_t *custom_message = reinterpret_cast<h2o_custom_res_message_t*>(message);- if(custom_message->type == SEND_RESPONSE_MSG) {- request_response* req_res = static_cast<request_response*>(custom_message->data);- custom_message->http_server->send_response(req_res->req, generator, *req_res->response);- delete req_res->response;- delete req_res;+ if(custom_message->http_server->message_handlers.count(custom_message->type) != 0) {+ auto handler = custom_message->http_server->message_handlers.at(custom_message->type);+ (handler)(custom_message->data);}h2o_linklist_unlink(&message->link);@@ -224,30 +217,17 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {}}- if(rpath.async) {- /*- Must call h2o_start_response and h2o_send functions within the same thread that- called the on_req callback.- */- std::thread response_thread([=]() {- http_req request = {query_map, req_body};- http_res* response = new http_res();- (rpath.handler)(request, *response);- request_response* req_res = new request_response{req, response};- h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{NULL}}, self->http_server,- SEND_RESPONSE_MSG, req_res};- h2o_multithread_send_message(self->http_server->message_receiver, &message->super);- });-- response_thread.detach();- return 0;+ http_req* request = new http_req{req, query_map, req_body};+ http_res* response = new http_res();+ response->server = self->http_server;+ (rpath.handler)(*request, *response);++ if(!rpath.async) {+ // If a handler is marked async, it's assumed that it's responsible for sending the response itself+ // later in an async fashion by calling into the main http thread via a message+ self->http_server->send_response(request, response);}- http_req request = {query_map, req_body};- http_res response;- h2o_generator_t generator = {NULL, NULL};- (rpath.handler)(request, response);- self->http_server->send_response(req, generator, response);return 0;}}@@ -263,13 +243,24 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {return 0;}-void HttpServer::send_response(h2o_req_t* req, h2o_generator_t & generator, const http_res & response) {- h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);- req->res.status = response.status_code;- req->res.reason = get_status_reason(response.status_code);+void HttpServer::send_message(const std::string & type, void* data) {+ h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{NULL}}, this, type.c_str(), data};+ h2o_multithread_send_message(message_receiver, &message->super);+}++void HttpServer::send_response(http_req* request, const http_res* response) {+ h2o_req_t* req = request->_req;+ h2o_generator_t generator = {NULL, NULL};++ h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);+ req->res.status = response->status_code;+ req->res.reason = get_status_reason(response->status_code);h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL, H2O_STRLIT("application/json; charset=utf-8"));h2o_start_response(req, &generator);h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);++ delete request;+ delete response;}int HttpServer::send_401_unauthorized(h2o_req_t *req) {@@ -312,6 +303,11 @@ void HttpServer::del(const std::string & path, void (*handler)(http_req &, http_routes.push_back(rpath);}++void HttpServer::on(const std::string & message, void (*handler)(void*)) {+ message_handlers.emplace(message, handler);+}+HttpServer::~HttpServer() {delete accept_ctx;delete message_receiver;diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 7f67f891..71f47e3a 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -1,8 +1,13 @@#include <signal.h>#include <iostream>+#include <vector>+#include <string>+#include <thread>#include <cmdline.h>#include "http_server.h"#include "api.h"+#include "string_utils.h"+#include "replicator.h"void catch_interrupt(int sig) {std::cout << "Quitting Typesense..." << std::endl;@@ -14,6 +19,8 @@ int main(int argc, char **argv) {options.add<std::string>("api-auth-key", 'k', "Key for authenticating the API endpoints.", true);options.add<std::string>("listen-address", 'a', "Address to which Typesense server binds.", false, "0.0.0.0");options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8080);+ options.add<std::string>("master", 'm', "<master_address>:<master_port> combination to start the server as a "+ "read-only replica.", false, "");options.parse_check(argc, argv);signal(SIGINT, catch_interrupt);@@ -42,6 +49,27 @@ int main(int argc, char **argv) {// replicationserver.get("/replication/updates", get_replication_updates, true, true);+ server.on(SEND_RESPONSE_MSG, on_send_response);+ server.on(REPLICATION_EVENT_MSG, Replicator::on_replication_event);++ // start a background replication thread if the server is started as a read-only replica+ if(!options.get<std::string>("master").empty()) {+ const std::string & master_host_port = options.get<std::string>("master");+ std::vector<std::string> parts;+ StringUtils::split(master_host_port, parts, ":");+ if(parts.size() != 2) {+ std::cerr << "Invalid value for --master option. Usage: <master_address>:<master_port>" << std::endl;+ return 1;+ }++ std::cout << "Typesense server started as a read-only replica... Spawning replication thread..." << std::endl;+ std::thread replication_thread([&server, &master_host_port, &store]() {+ Replicator::start(&server, master_host_port, store);+ });++ replication_thread.detach();+ }+server.run();return 0;}\ No newline at end of file