Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
API for fetching all transactions from a given transaction sequence number.
Kishore Nallan6a465a02898b
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 3385c774..6094b8fe 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -70,7 +70,7 @@ add_executable(search ${SRC_FILES} src/main/main.cpp)add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpptest/collection_test.cpp test/collection_manager_test.cpp- test/topster_test.cpp test/match_score_test.cpp)+ test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp)target_compile_definitions(typesense-server PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")target_compile_definitions(search PRIVATE ROOT_DIR="${CMAKE_SOURCE_DIR}/")diff --git a/TODO.md b/TODO.mdindex e951d864..57361200 100644--- a/TODO.md+++ b/TODO.md@@ -57,6 +57,9 @@- ~~only last token should be prefix searched~~- ~~Prefix-search strings should not be null terminated~~- ~~sort results by float field~~+- Validate before string to int conversion in the http api layer+- When field of "id" but not string, what happens?+- Typo in prefix search- test for string filter comparison: title < "foo"- test for token ranking on float field- test for float int field deletion during doc deletion@@ -73,6 +76,7 @@- Test for sorted_array::indexOf when length is 0- Test for snippets- Test for pagination+- Test for string utils- Prevent string copy during indexing- clean special chars before indexing- Minimum results should be a variable instead of blindly going with max_resultsdiff --git a/include/api.h b/include/api.hindex 24f8352c..1a88181b 100644--- a/include/api.h+++ b/include/api.h@@ -16,4 +16,6 @@ void post_add_document(http_req & req, http_res & res);void get_fetch_document(http_req & req, http_res & res);-void del_remove_document(http_req & req, http_res & res);\ No newline at end of file+void del_remove_document(http_req & req, http_res & res);++void get_replication_updates(http_req &req, http_res &res);\ No newline at end of filediff --git a/include/collection_manager.h b/include/collection_manager.hindex e67a56f8..b4f53ec2 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -57,5 +57,7 @@ public:uint32_t get_next_collection_id();+ Store* get_store();+static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";};\ No newline at end of filediff --git a/include/store.h b/include/store.hindex da801488..482db444 100644--- a/include/store.h+++ b/include/store.h@@ -3,9 +3,12 @@#include <stdint.h>#include <cstdlib>#include <string>+#include <memory>+#include <option.h>#include <rocksdb/db.h>#include <rocksdb/options.h>#include <rocksdb/merge_operator.h>+#include <rocksdb/transaction_log.h>class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {public:@@ -56,6 +59,10 @@ public:options.max_write_buffer_number = 2;options.merge_operator.reset(new UInt64AddOperator);+ // these need to be high for replication scenarios+ options.WAL_ttl_seconds = 24*60*60;+ options.WAL_size_limit_MB = 1024;+// open DBrocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);if(!s.ok()) {@@ -122,11 +129,38 @@ public:db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));}+ Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number, const uint64_t max_updates) {+ Option<std::vector<std::string>*> updates_op(new std::vector<std::string>());++ rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;+ rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);++ if(!status.ok()) {+ return Option<std::vector<std::string>*>(500, "Could not fetch updates from the disk store.");+ }++ uint64_t num_updates = 0;+ while(iter->Valid() && num_updates < max_updates) {+ rocksdb::BatchResult batch_result = iter->GetBatch();+ const std::string & write_batch_serialized = batch_result.writeBatchPtr->Data();+ updates_op.get()->push_back(write_batch_serialized);+ num_updates += 1;+ iter->Next();+ }++ return updates_op;+ }+void close() {delete db;db = nullptr;}+ // Only for internal tests+ rocksdb::DB* _get_db_unsafe() {+ return db;+ }+void print_memory_usage() {std::string index_usage;db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);diff --git a/include/string_utils.h b/include/string_utils.hindex ffd9f236..f29578fa 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -105,7 +105,65 @@ struct StringUtils {return (*p == 0);}+ // Adapted from: http://stackoverflow.com/a/2845275/131050+ static bool is_uint64_t(const std::string &s) {+ if(s.empty()) {+ return false;+ }++ char * p ;+ strtoull(s.c_str(), &p, 10);+ return (*p == 0);+ }+static void toupper(std::string& str) {std::transform(str.begin(), str.end(), str.begin(), ::toupper);}++ /* https://stackoverflow.com/a/34571089/131050 */+ static std::string base64_encode(const std::string &in) {+ std::string out;++ int val=0, valb=-6;+ for(unsigned char c : in) {+ val = (val<<8) + c;+ valb += 8;+ while (valb>=0) {+ out.push_back("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(val>>valb)&0x3F]);+ valb-=6;+ }+ }++ if(valb>-6) {+ out.push_back("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[((val<<8)>>(valb+8))&0x3F]);+ }+ while(out.size()%4) {+ out.push_back('=');+ }++ return out;+ }++ static std::string base64_decode(const std::string &in) {+ std::string out;++ std::vector<int> T(256,-1);+ for(int i=0; i<64; i++) {+ T["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[i]] = i;+ }++ int val=0, valb=-8;++ for(unsigned char c : in) {+ if (T[c] == -1) break;+ val = (val<<6) + T[c];+ valb += 6;+ if (valb>=0) {+ out.push_back(char((val>>valb)&0xFF));+ valb-=8;+ }+ }++ return out;+ }};\ No newline at end of filediff --git a/src/api.cpp b/src/api.cppindex 695c042a..fa36faa4 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -332,4 +332,30 @@ void del_remove_document(http_req & req, http_res & res) {json_response["id"] = deleted_id_op.get();res.send_200(json_response.dump());}+}++void get_replication_updates(http_req &req, http_res &res) {+ if(!StringUtils::is_uint64_t(req.params["seq_number"])) {+ return res.send_400("The value of the parameter `seq_number` must be an unsigned integer.");+ }++ const uint64_t MAX_UPDATES_TO_SEND = 2000;+ uint64_t seq_number = std::stoull(req.params["seq_number"]);++ CollectionManager & collectionManager = CollectionManager::get_instance();+ Store* store = collectionManager.get_store();+ Option<std::vector<std::string>*> updates_op = store->get_updates_since(seq_number, MAX_UPDATES_TO_SEND);+ if(!updates_op.ok()) {+ return res.send(updates_op.code(), updates_op.error());+ }++ nlohmann::json json_response;+ json_response["updates"] = nlohmann::json::array();++ std::vector<std::string> *updates = updates_op.get();+ for(const std::string & update: *updates) {+ json_response["updates"].push_back(StringUtils::base64_encode(update));+ }++ res.send_200(json_response.dump());}\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex dd593092..9371210c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -195,4 +195,8 @@ Option<bool> CollectionManager::drop_collection(std::string collection_name) {uint32_t CollectionManager::get_next_collection_id() {return next_collection_id;+}++Store* CollectionManager::get_store() {+ return store;}\ No newline at end of filediff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex c97b8b74..03520aee 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -27,7 +27,7 @@ int main(int argc, char* argv[]) {collection = collectionManager.create_collection("hnstories_direct", fields_to_index, {}, sort_fields, "points");}- std::ifstream infile("/Users/kishore/Downloads/hnstories.jsonl");+ std::ifstream infile("/Users/kishore/Downloads/hnstories_tiny.jsonl");std::string json_line;auto begin = std::chrono::high_resolution_clock::now();diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex bb1bb66c..502d010b 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -39,6 +39,9 @@ int main(int argc, char **argv) {server.get("/collections/:collection/:id", get_fetch_document, true);server.del("/collections/:collection/:id", del_remove_document, true);+ // replication+ server.get("/replication/updates", get_replication_updates, true);+server.run();return 0;}\ No newline at end of filediff --git a/test/store_test.cpp b/test/store_test.cppnew file mode 100644index 00000000..8608c54c--- /dev/null+++ b/test/store_test.cpp@@ -0,0 +1,47 @@+#include <gtest/gtest.h>+#include <vector>+#include <store.h>+#include <string_utils.h>++TEST(StoreTest, GetUpdatesSince) {+ std::string primary_store_path = "/tmp/typesense_test/primary_store_test";+ std::cout << "Truncating and creating: " << primary_store_path << std::endl;+ system(("rm -rf "+primary_store_path+" && mkdir -p "+primary_store_path).c_str());++ // add some records, get the updates and restore them in a new store+ Store primary_store(primary_store_path);+ primary_store.insert("foo1", "bar1");+ primary_store.insert("foo2", "bar2");+ primary_store.insert("foo3", "bar3");++ Option<std::vector<std::string>*> updates_op = primary_store.get_updates_since(0, 10);+ ASSERT_EQ(3, updates_op.get()->size());++ std::string replica_store_path = "/tmp/typesense_test/replica_store_test";+ std::cout << "Truncating and creating: " << replica_store_path << std::endl;+ system(("rm -rf "+replica_store_path+" && mkdir -p "+replica_store_path).c_str());++ Store replica_store(replica_store_path);+ rocksdb::DB* replica_db = replica_store._get_db_unsafe();++ for(const std::string & update: *updates_op.get()) {+ // Do Base64 encoding and decoding as we would in the API layer+ const std::string update_encoded = StringUtils::base64_encode(update);+ const std::string update_decoded = StringUtils::base64_decode(update_encoded);+ rocksdb::WriteBatch write_batch(update_decoded);+ replica_db->Write(rocksdb::WriteOptions(), &write_batch);+ }++ std::string value;+ for(auto i=1; i<=3; i++) {+ replica_store.get(std::string("foo")+std::to_string(i), value);+ ASSERT_EQ(std::string("bar")+std::to_string(i), value);+ }++ delete updates_op.get();++ // Ensure that updates are limited to max_updates argument+ updates_op = primary_store.get_updates_since(0, 2);+ ASSERT_EQ(2, updates_op.get()->size());+ delete updates_op.get();+}\ No newline at end of file