Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle concurrent access of DB handle.
kishorencbb530f259efa
diff --git a/.circleci/config.yml b/.circleci/config.ymlindex da51f931..59e120e2 100644--- a/.circleci/config.yml+++ b/.circleci/config.yml@@ -2,7 +2,7 @@ version: 2jobs:build:docker:- - image: typesense/typesense-development:25-JANUARY-2021-1+ - image: typesense/typesense-development:05-FEBRUARY-2021-1environment:- PROJECT_DIR: /typesense- TYPESENSE_VERSION: $CIRCLE_BRANCH-$CIRCLE_SHA1@@ -19,7 +19,7 @@ jobs:- external-Linux-cache- run:name: build- command: $PROJECT_DIR/build.sh+ command: $PROJECT_DIR/build.sh --clean- save_cache:key: external-Linux-cache-{{ .Branch }}-{{ checksum "last-changed-git-sha-for-dependency-listing" }}paths:diff --git a/CMakeLists.txt b/CMakeLists.txtindex 4e147b41..da7aa09f 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -3,11 +3,16 @@ project(typesense)cmake_policy(SET CMP0074 NEW)-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++11 -O0 -g")+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++17 -O0 -g")set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess")+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-class-memaccess")+endif()+# will be overridden from command line during buildset(TYPESENSE_VERSION "nightly" CACHE STRING "")set(BUILD_DEPS "yes" CACHE STRING "")diff --git a/docker-build.sh b/docker-build.shindex 5bab1fe5..59b81e07 100755--- a/docker-build.sh+++ b/docker-build.sh@@ -22,10 +22,10 @@ if [[ "$@" == *"--depclean"* ]]; thenfiecho "Building Typesense $TYPESENSE_VERSION..."-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \-DCMAKE_BUILD_TYPE=Release -H/typesense -B/typesense/$BUILD_DIR-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIR+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIRif [[ "$@" == *"--build-deploy-image"* ]]; thenecho "Creating deployment image for Typesense $TYPESENSE_VERSION server ..."@@ -45,7 +45,7 @@ fi##if [[ "$@" == *"--create-deb-upload"* ]]; then-# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \+# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \# -DCMAKE_BUILD_TYPE=Debug -H/typesense -B/typesense/$BUILD_DIR#fidiff --git a/include/store.h b/include/store.hindex b700df0d..43dc1a1f 100644--- a/include/store.h+++ b/include/store.h@@ -5,6 +5,8 @@#include <string>#include <sstream>#include <memory>+#include <thread>+#include <shared_mutex>#include <option.h>#include <rocksdb/db.h>#include <rocksdb/write_batch.h>@@ -51,6 +53,10 @@ private:rocksdb::Options options;rocksdb::WriteOptions write_options;+ // Used to protect assignment to DB handle, which is otherwise thread safe+ // So we use unique lock only for assignment, but shared locks for all other operations on DB+ mutable std::shared_mutex mutex;+rocksdb::Status init_db() {rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);if(!s.ok()) {@@ -102,16 +108,20 @@ public:}bool insert(const std::string& key, const std::string& value) {+ std::shared_lock lock(mutex);rocksdb::Status status = db->Put(write_options, key, value);return status.ok();}bool batch_write(rocksdb::WriteBatch& batch) {+ std::shared_lock lock(mutex);rocksdb::Status status = db->Write(write_options, &batch);return status.ok();}bool contains(const std::string& key) const {+ std::shared_lock lock(mutex);+std::string value;bool value_found;bool key_may_exist = db->KeyMayExist(rocksdb::ReadOptions(), key, &value, &value_found);@@ -131,6 +141,7 @@ public:}StoreStatus get(const std::string& key, std::string& value) const {+ std::shared_lock lock(mutex);rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);if(status.ok()) {@@ -146,22 +157,26 @@ public:}bool remove(const std::string& key) {+ std::shared_lock lock(mutex);rocksdb::Status status = db->Delete(write_options, key);return status.ok();}rocksdb::Iterator* scan(const std::string & prefix) {+ std::shared_lock lock(mutex);rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());iter->Seek(prefix);return iter;}rocksdb::Iterator* get_iterator() {+ std::shared_lock lock(mutex);rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());return it;};void scan_fill(const std::string & prefix, std::vector<std::string> & values) {+ std::shared_lock lock(mutex);rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {values.push_back(iter->value().ToString());@@ -171,14 +186,17 @@ public:}void increment(const std::string & key, uint32_t value) {+ std::shared_lock lock(mutex);db->Merge(write_options, key, StringUtils::serialize_uint32_t(value));}uint64_t get_latest_seq_number() const {+ std::shared_lock lock(mutex);return db->GetLatestSequenceNumber();}Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const {+ std::shared_lock lock(mutex);const uint64_t local_latest_seq_num = db->GetLatestSequenceNumber();// Since GetUpdatesSince(0) == GetUpdatesSince(1)@@ -242,12 +260,17 @@ public:}void close() {+ std::unique_lock lock(mutex);delete db;db = nullptr;}int reload(bool clear_state_dir, const std::string& snapshot_path) {- close();+ std::unique_lock lock(mutex);++ // we don't use close() to avoid nested lock and because lock is required until db is re-initialized+ delete db;+ db = nullptr;if(clear_state_dir) {if (!butil::DeleteFile(butil::FilePath(state_dir_path), true)) {@@ -285,11 +308,13 @@ public:}void flush() {+ std::shared_lock lock(mutex);rocksdb::FlushOptions options;db->Flush(options);}rocksdb::Status create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path) {+ std::shared_lock lock(mutex);rocksdb::Status status = rocksdb::Checkpoint::Create(db, checkpoint_ptr);if(!status.ok()) {LOG(ERROR) << "Checkpoint Create failed, msg:" << status.ToString();diff --git a/src/raft_server.cpp b/src/raft_server.cppindex f3afaf31..ea709224 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -405,6 +405,9 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {LOG(INFO) << "on_snapshot_load";+ // ensures that reads are rejected, as `store->reload()` unique locks the DB handle+ caught_up = false;+// Load snapshot from leader, replacing the running StateMachinestd::string snapshot_path = reader->get_path();snapshot_path.append(std::string("/") + db_snapshot_name);