Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

API for fetching all transactions from a given transaction sequence number.

Kishore Nallan6a465a02898b
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3385c774..6094b8fe 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,7 +70,7 @@ add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
test/collection_test.cpp test/collection_manager_test.cpp
- test/topster_test.cpp test/match_score_test.cpp)
+ test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp)
target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")
diff --git a/TODO.md b/TODO.md
index e951d864..57361200 100644
--- a/TODO.md
+++ b/TODO.md
@@ -57,6 +57,9 @@
- ~~only last token should be prefix searched~~
- ~~Prefix-search strings should not be null terminated~~
- ~~sort results by float field~~
+- Validate before string to int conversion in the http api layer
+- When field of "id" but not string, what happens?
+- Typo in prefix search
- test for string filter comparison: title < "foo"
- test for token ranking on float field
- test for float int field deletion during doc deletion
@@ -73,6 +76,7 @@
- Test for sorted_array::indexOf when length is 0
- Test for snippets
- Test for pagination
+- Test for string utils
- Prevent string copy during indexing
- clean special chars before indexing
- Minimum results should be a variable instead of blindly going with max_results
diff --git a/include/api.h b/include/api.h
index 24f8352c..1a88181b 100644
--- a/include/api.h
+++ b/include/api.h
@@ -16,4 +16,6 @@ void post_add_document(http_req & req, http_res & res);
void get_fetch_document(http_req & req, http_res & res);
-void del_remove_document(http_req & req, http_res & res);
\ No newline at end of file
+void del_remove_document(http_req & req, http_res & res);
+
+void get_replication_updates(http_req &req, http_res &res);
\ No newline at end of file
diff --git a/include/collection_manager.h b/include/collection_manager.h
index e67a56f8..b4f53ec2 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -57,5 +57,7 @@ public:
uint32_t get_next_collection_id();
+ Store* get_store();
+
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
};
\ No newline at end of file
diff --git a/include/store.h b/include/store.h
index da801488..482db444 100644
--- a/include/store.h
+++ b/include/store.h
@@ -3,9 +3,12 @@
#include <stdint.h>
#include <cstdlib>
#include <string>
+#include <memory>
+#include <option.h>
#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <rocksdb/merge_operator.h>
+#include <rocksdb/transaction_log.h>
class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {
public:
@@ -56,6 +59,10 @@ public:
options.max_write_buffer_number = 2;
options.merge_operator.reset(new UInt64AddOperator);
+ // these need to be high for replication scenarios
+ options.WAL_ttl_seconds = 24*60*60;
+ options.WAL_size_limit_MB = 1024;
+
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
if(!s.ok()) {
@@ -122,11 +129,38 @@ public:
db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));
}
+ Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number, const uint64_t max_updates) {
+ Option<std::vector<std::string>*> updates_op(new std::vector<std::string>());
+
+ rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;
+ rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
+
+ if(!status.ok()) {
+ return Option<std::vector<std::string>*>(500, "Could not fetch updates from the disk store.");
+ }
+
+ uint64_t num_updates = 0;
+ while(iter->Valid() && num_updates < max_updates) {
+ rocksdb::BatchResult batch_result = iter->GetBatch();
+ const std::string & write_batch_serialized = batch_result.writeBatchPtr->Data();
+ updates_op.get()->push_back(write_batch_serialized);
+ num_updates += 1;
+ iter->Next();
+ }
+
+ return updates_op;
+ }
+
void close() {
delete db;
db = nullptr;
}
+ // Only for internal tests
+ rocksdb::DB* _get_db_unsafe() {
+ return db;
+ }
+
void print_memory_usage() {
std::string index_usage;
db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
diff --git a/include/string_utils.h b/include/string_utils.h
index ffd9f236..f29578fa 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -105,7 +105,65 @@ struct StringUtils {
return (*p == 0);
}
+ // Adapted from: http://stackoverflow.com/a/2845275/131050
+ static bool is_uint64_t(const std::string &s) {
+ if(s.empty()) {
+ return false;
+ }
+
+ char * p ;
+ strtoull(s.c_str(), &p, 10);
+ return (*p == 0);
+ }
+
static void toupper(std::string& str) {
std::transform(str.begin(), str.end(), str.begin(), ::toupper);
}
+
+ /* https://stackoverflow.com/a/34571089/131050 */
+ static std::string base64_encode(const std::string &in) {
+ std::string out;
+
+ int val=0, valb=-6;
+ for(unsigned char c : in) {
+ val = (val<<8) + c;
+ valb += 8;
+ while (valb>=0) {
+ out.push_back("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(val>>valb)&0x3F]);
+ valb-=6;
+ }
+ }
+
+ if(valb>-6) {
+ out.push_back("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[((val<<8)>>(valb+8))&0x3F]);
+ }
+ while(out.size()%4) {
+ out.push_back('=');
+ }
+
+ return out;
+ }
+
+ static std::string base64_decode(const std::string &in) {
+ std::string out;
+
+ std::vector<int> T(256,-1);
+ for(int i=0; i<64; i++) {
+ T["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[i]] = i;
+ }
+
+ int val=0, valb=-8;
+
+ for(unsigned char c : in) {
+ if (T[c] == -1) break;
+ val = (val<<6) + T[c];
+ valb += 6;
+ if (valb>=0) {
+ out.push_back(char((val>>valb)&0xFF));
+ valb-=8;
+ }
+ }
+
+ return out;
+ }
};
\ No newline at end of file
diff --git a/src/api.cpp b/src/api.cpp
index 695c042a..fa36faa4 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -332,4 +332,30 @@ void del_remove_document(http_req & req, http_res & res) {
json_response["id"] = deleted_id_op.get();
res.send_200(json_response.dump());
}
+}
+
+void get_replication_updates(http_req &req, http_res &res) {
+ if(!StringUtils::is_uint64_t(req.params["seq_number"])) {
+ return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");
+ }
+
+ const uint64_t MAX_UPDATES_TO_SEND = 2000;
+ uint64_t seq_number = std::stoull(req.params["seq_number"]);
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Store* store = collectionManager.get_store();
+ Option<std::vector<std::string>*> updates_op = store->get_updates_since(seq_number, MAX_UPDATES_TO_SEND);
+ if(!updates_op.ok()) {
+ return res.send(updates_op.code(), updates_op.error());
+ }
+
+ nlohmann::json json_response;
+ json_response["updates"] = nlohmann::json::array();
+
+ std::vector<std::string> *updates = updates_op.get();
+ for(const std::string & update: *updates) {
+ json_response["updates"].push_back(StringUtils::base64_encode(update));
+ }
+
+ res.send_200(json_response.dump());
}
\ No newline at end of file
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index dd593092..9371210c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -195,4 +195,8 @@ Option<bool> CollectionManager::drop_collection(std::string collection_name) {
uint32_t CollectionManager::get_next_collection_id() {
return next_collection_id;
+}
+
+Store* CollectionManager::get_store() {
+ return store;
}
\ No newline at end of file
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index c97b8b74..03520aee 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -27,7 +27,7 @@ int main(int argc, char* argv[]) {
collection = collectionManager.create_collection("hnstories_direct", fields_to_index, {}, sort_fields, "points");
}
- std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");
+ std::ifstream infile("/Users/kishore/Downloads/hnstories_tiny.jsonl");
std::string json_line;
auto begin = std::chrono::high_resolution_clock::now();
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index bb1bb66c..502d010b 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -39,6 +39,9 @@ int main(int argc, char **argv) {
server.get("/collections/:collection/:id", get_fetch_document, true);
server.del("/collections/:collection/:id", del_remove_document, true);
+ // replication
+ server.get("/replication/updates", get_replication_updates, true);
+
server.run();
return 0;
}
\ No newline at end of file
diff --git a/test/store_test.cpp b/test/store_test.cpp
new file mode 100644
index 00000000..8608c54c
--- /dev/null
+++ b/test/store_test.cpp
@@ -0,0 +1,47 @@
+#include <gtest/gtest.h>
+#include <vector>
+#include <store.h>
+#include <string_utils.h>
+
+TEST(StoreTest, GetUpdatesSince) {
+ std::string primary_store_path = "/tmp/typesense_test/primary_store_test";
+ std::cout << "Truncating and creating: " << primary_store_path << std::endl;
+ system(("rm -rf "+primary_store_path+" && mkdir -p "+primary_store_path).c_str());
+
+ // add some records, get the updates and restore them in a new store
+ Store primary_store(primary_store_path);
+ primary_store.insert("foo1", "bar1");
+ primary_store.insert("foo2", "bar2");
+ primary_store.insert("foo3", "bar3");
+
+ Option<std::vector<std::string>*> updates_op = primary_store.get_updates_since(0, 10);
+ ASSERT_EQ(3, updates_op.get()->size());
+
+ std::string replica_store_path = "/tmp/typesense_test/replica_store_test";
+ std::cout << "Truncating and creating: " << replica_store_path << std::endl;
+ system(("rm -rf "+replica_store_path+" && mkdir -p "+replica_store_path).c_str());
+
+ Store replica_store(replica_store_path);
+ rocksdb::DB* replica_db = replica_store._get_db_unsafe();
+
+ for(const std::string & update: *updates_op.get()) {
+ // Do Base64 encoding and decoding as we would in the API layer
+ const std::string update_encoded = StringUtils::base64_encode(update);
+ const std::string update_decoded = StringUtils::base64_decode(update_encoded);
+ rocksdb::WriteBatch write_batch(update_decoded);
+ replica_db->Write(rocksdb::WriteOptions(), &write_batch);
+ }
+
+ std::string value;
+ for(auto i=1; i<=3; i++) {
+ replica_store.get(std::string("foo")+std::to_string(i), value);
+ ASSERT_EQ(std::string("bar")+std::to_string(i), value);
+ }
+
+ delete updates_op.get();
+
+ // Ensure that updates are limited to max_updates argument
+ updates_op = primary_store.get_updates_since(0, 2);
+ ASSERT_EQ(2, updates_op.get()->size());
+ delete updates_op.get();
+}
\ No newline at end of file