Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Improve replica's logging - make it more useful.

Kishore Nallan0a9e5e15625c
diff --git a/TODO.md b/TODO.md
index a4dc7486..199fdf25 100644
--- a/TODO.md
+++ b/TODO.md
@@ -92,6 +92,9 @@
- ~~Handle store-get() not finding a key~~
- ~~Deprecate converting integer to string verbatim~~
- ~~Deprecate union type punning~~
+- Replica server should fail when pointed to "old" master
+- Parameterize replica's MAX_UPDATES_TO_SEND
+- gzip compress responses
- NOT operator support
- > INT32_MAX validation for float field
- highlight of string arrays?
diff --git a/include/replicator.h b/include/replicator.h
index 3e58c870..bc8b56c7 100644
--- a/include/replicator.h
+++ b/include/replicator.h
@@ -90,10 +90,16 @@ public:
class Replicator {
public:
static void start(HttpServer* server, const std::string master_host_port, const std::string api_key, Store& store) {
+ size_t total_runs = 0;
+
while(true) {
IterateBatchHandler handler(server);
uint64_t latest_seq_num = store.get_latest_seq_number();
- LOG(INFO) << "latest_seq_num: " << latest_seq_num;
+
+ if(total_runs++ % 20 == 0) {
+ // roughly every 60 seconds
+ LOG(INFO) << "Replica's latest sequence number: " << latest_seq_num;
+ }
HttpClient client(
master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1), api_key
@@ -105,17 +111,26 @@ public:
if(status_code == 200) {
nlohmann::json json_content = nlohmann::json::parse(json_response);
nlohmann::json updates = json_content["updates"];
+
+ // first write to memory
for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {
const std::string update_decoded = StringUtils::base64_decode(*update);
rocksdb::WriteBatch write_batch(update_decoded);
write_batch.Iterate(&handler);
}
+ // now write to store
for (nlohmann::json::iterator update = updates.begin(); update != updates.end(); ++update) {
const std::string update_decoded = StringUtils::base64_decode(*update);
rocksdb::WriteBatch write_batch(update_decoded);
store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);
}
+
+ if(updates.size() > 0) {
+ LOG(INFO) << "Replica has consumed " << latest_seq_num+updates.size() << "/"
+ << json_content["latest_seq_num"] << " updates from master.";
+ }
+
} else {
LOG(ERR) << "Replication error while fetching records from master, status_code=" << status_code;
diff --git a/src/api.cpp b/src/api.cpp
index 061dab33..ba531143 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -412,7 +412,7 @@ void get_replication_updates(http_req & req, http_res & res) {
return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");
}
- const uint64_t MAX_UPDATES_TO_SEND = 2000;
+ const uint64_t MAX_UPDATES_TO_SEND = 10000;
uint64_t seq_number = std::stoull(req.params["seq_number"]);
CollectionManager & collectionManager = CollectionManager::get_instance();
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index ccf1dcf1..1077b487 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -28,7 +28,7 @@ int main(int argc, char* argv[]) {
collection = collectionManager.create_collection("hnstories_direct", fields_to_index, "points").get();
}
- std::ifstream infile("/Users/kishore/Downloads/hnstories_tiny.jsonl");
+ std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
std::string json_line;
auto begin = std::chrono::high_resolution_clock::now();
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 66a2623f..2dd93fbc 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -113,6 +113,8 @@ int main(int argc, char **argv) {
return 1;
}
+ LOG(INFO) << "Loading collections from disk...";
+
Store store(options.get<std::string>("data-dir"));
CollectionManager & collectionManager = CollectionManager::get_instance();
Option<bool> init_op = collectionManager.init(&store, options.get<std::string>("api-key"),