Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Improve replica's logging - make it more useful.
Kishore Nallan0a9e5e15625c
diff --git a/TODO.md b/TODO.mdindex a4dc7486..199fdf25 100644--- a/TODO.md+++ b/TODO.md@@ -92,6 +92,9 @@- ~~Handle store-get() not finding a key~~- ~~Deprecate converting integer to string verbatim~~- ~~Deprecate union type punning~~+- Replica server should fail when pointed to "old" master+- Parameterize replica's MAX_UPDATES_TO_SEND+- gzip compress responses- NOT operator support- > INT32_MAX validation for float field- highlight of string arrays?diff --git a/include/replicator.h b/include/replicator.hindex 3e58c870..bc8b56c7 100644--- a/include/replicator.h+++ b/include/replicator.h@@ -90,10 +90,16 @@ public:class Replicator {public:static void start(HttpServer* server, const std::string master_host_port, const std::string api_key, Store& store) {+ size_t total_runs = 0;+while(true) {IterateBatchHandler handler(server);uint64_t latest_seq_num = store.get_latest_seq_number();- LOG(INFO) << "latest_seq_num: " << latest_seq_num;++ if(total_runs++ % 20 == 0) {+ // roughly every 60 seconds+ LOG(INFO) << "Replica's latest sequence number: " << latest_seq_num;+ }HttpClient client(master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1), api_key@@ -105,17 +111,26 @@ public:if(status_code == 200) {nlohmann::json json_content = nlohmann::json::parse(json_response);nlohmann::json updates = json_content["updates"];++ // first write to memoryfor (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {const std::string update_decoded = StringUtils::base64_decode(*update);rocksdb::WriteBatch write_batch(update_decoded);write_batch.Iterate(&handler);}+ // now write to storefor (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {const std::string update_decoded = StringUtils::base64_decode(*update);rocksdb::WriteBatch write_batch(update_decoded);store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);}++ if(updates.size() > 0) {+ LOG(INFO) << "Replica has consumed " << latest_seq_num+updates.size() << "/"+ << json_content["latest_seq_num"] << " updates from master.";+ }+} else {LOG(ERR) << "Replication error while fetching records from master, status_code=" << status_code;diff --git a/src/api.cpp b/src/api.cppindex 061dab33..ba531143 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -412,7 +412,7 @@ void get_replication_updates(http_req & req, http_res & res) {return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");}- const uint64_t MAX_UPDATES_TO_SEND = 2000;+ const uint64_t MAX_UPDATES_TO_SEND = 10000;uint64_t seq_number = std::stoull(req.params["seq_number"]);CollectionManager & collectionManager = CollectionManager::get_instance();diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex ccf1dcf1..1077b487 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -28,7 +28,7 @@ int main(int argc, char* argv[]) {collection = collectionManager.create_collection("hnstories_direct", fields_to_index, "points").get();}- std::ifstream infile("/Users/kishore/Downloads/hnstories_tiny.jsonl");+ std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");std::string json_line;auto begin = std::chrono::high_resolution_clock::now();diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 66a2623f..2dd93fbc 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -113,6 +113,8 @@ int main(int argc, char **argv) {return 1;}+ LOG(INFO) << "Loading collections from disk...";+Store store(options.get<std::string>("data-dir"));CollectionManager & collectionManager = CollectionManager::get_instance();Option<bool> init_op = collectionManager.init(&store, options.get<std::string>("api-key"),