Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle concurrent access of DB handle.

kishorencbb530f259efa
diff --git a/.circleci/config.yml b/.circleci/config.yml
index da51f931..59e120e2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
jobs:
build:
docker:
- - image: typesense/typesense-development:25-JANUARY-2021-1
+ - image: typesense/typesense-development:05-FEBRUARY-2021-1
environment:
- PROJECT_DIR: /typesense
- TYPESENSE_VERSION: $CIRCLE_BRANCH-$CIRCLE_SHA1
@@ -19,7 +19,7 @@ jobs:
- external-Linux-cache
- run:
name: build
- command: $PROJECT_DIR/build.sh
+ command: $PROJECT_DIR/build.sh --clean
- save_cache:
key: external-Linux-cache-{{ .Branch }}-{{ checksum "last-changed-git-sha-for-dependency-listing" }}
paths:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e147b41..da7aa09f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,11 +3,16 @@ project(typesense)
cmake_policy(SET CMP0074 NEW)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++11 -O0 -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++17 -O0 -g")
set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-class-memaccess")
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-class-memaccess")
+endif()
+
# will be overridden from command line during build
set(TYPESENSE_VERSION "nightly" CACHE STRING "")
set(BUILD_DEPS "yes" CACHE STRING "")
diff --git a/docker-build.sh b/docker-build.sh
index 5bab1fe5..59b81e07 100755
--- a/docker-build.sh
+++ b/docker-build.sh
@@ -22,10 +22,10 @@ if [[ "$@" == *"--depclean"* ]]; then
fi
echo "Building Typesense $TYPESENSE_VERSION..."
-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
-DCMAKE_BUILD_TYPE=Release -H/typesense -B/typesense/$BUILD_DIR
-docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIR
+docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 make typesense-server -C/typesense/$BUILD_DIR
if [[ "$@" == *"--build-deploy-image"* ]]; then
echo "Creating deployment image for Typesense $TYPESENSE_VERSION server ..."
@@ -45,7 +45,7 @@ fi
#
#if [[ "$@" == *"--create-deb-upload"* ]]; then
-# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:25-JANUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
+# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:05-FEBRUARY-2021-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \
# -DCMAKE_BUILD_TYPE=Debug -H/typesense -B/typesense/$BUILD_DIR
#fi
diff --git a/include/store.h b/include/store.h
index b700df0d..43dc1a1f 100644
--- a/include/store.h
+++ b/include/store.h
@@ -5,6 +5,8 @@
#include <string>
#include <sstream>
#include <memory>
+#include <thread>
+#include <shared_mutex>
#include <option.h>
#include <rocksdb/db.h>
#include <rocksdb/write_batch.h>
@@ -51,6 +53,10 @@ private:
rocksdb::Options options;
rocksdb::WriteOptions write_options;
+ // Used to protect assignment to DB handle, which is otherwise thread safe
+ // So we use unique lock only for assignment, but shared locks for all other operations on DB
+ mutable std::shared_mutex mutex;
+
rocksdb::Status init_db() {
rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
if(!s.ok()) {
@@ -102,16 +108,20 @@ public:
}
bool insert(const std::string& key, const std::string& value) {
+ std::shared_lock lock(mutex);
rocksdb::Status status = db->Put(write_options, key, value);
return status.ok();
}
bool batch_write(rocksdb::WriteBatch& batch) {
+ std::shared_lock lock(mutex);
rocksdb::Status status = db->Write(write_options, &batch);
return status.ok();
}
bool contains(const std::string& key) const {
+ std::shared_lock lock(mutex);
+
std::string value;
bool value_found;
bool key_may_exist = db->KeyMayExist(rocksdb::ReadOptions(), key, &value, &value_found);
@@ -131,6 +141,7 @@ public:
}
StoreStatus get(const std::string& key, std::string& value) const {
+ std::shared_lock lock(mutex);
rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
if(status.ok()) {
@@ -146,22 +157,26 @@ public:
}
bool remove(const std::string& key) {
+ std::shared_lock lock(mutex);
rocksdb::Status status = db->Delete(write_options, key);
return status.ok();
}
rocksdb::Iterator* scan(const std::string & prefix) {
+ std::shared_lock lock(mutex);
rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());
iter->Seek(prefix);
return iter;
}
rocksdb::Iterator* get_iterator() {
+ std::shared_lock lock(mutex);
rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());
return it;
};
void scan_fill(const std::string & prefix, std::vector<std::string> & values) {
+ std::shared_lock lock(mutex);
rocksdb::Iterator *iter = db->NewIterator(rocksdb::ReadOptions());
for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {
values.push_back(iter->value().ToString());
@@ -171,14 +186,17 @@ public:
}
void increment(const std::string & key, uint32_t value) {
+ std::shared_lock lock(mutex);
db->Merge(write_options, key, StringUtils::serialize_uint32_t(value));
}
uint64_t get_latest_seq_number() const {
+ std::shared_lock lock(mutex);
return db->GetLatestSequenceNumber();
}
Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const {
+ std::shared_lock lock(mutex);
const uint64_t local_latest_seq_num = db->GetLatestSequenceNumber();
// Since GetUpdatesSince(0) == GetUpdatesSince(1)
@@ -242,12 +260,17 @@ public:
}
void close() {
+ std::unique_lock lock(mutex);
delete db;
db = nullptr;
}
int reload(bool clear_state_dir, const std::string& snapshot_path) {
- close();
+ std::unique_lock lock(mutex);
+
+ // we don't use close() to avoid nested lock and because lock is required until db is re-initialized
+ delete db;
+ db = nullptr;
if(clear_state_dir) {
if (!butil::DeleteFile(butil::FilePath(state_dir_path), true)) {
@@ -285,11 +308,13 @@ public:
}
void flush() {
+ std::shared_lock lock(mutex);
rocksdb::FlushOptions options;
db->Flush(options);
}
rocksdb::Status create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path) {
+ std::shared_lock lock(mutex);
rocksdb::Status status = rocksdb::Checkpoint::Create(db, checkpoint_ptr);
if(!status.ok()) {
LOG(ERROR) << "Checkpoint Create failed, msg:" << status.ToString();
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index f3afaf31..ea709224 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -405,6 +405,9 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
LOG(INFO) << "on_snapshot_load";
+ // ensures that reads are rejected, as `store->reload()` unique locks the DB handle
+ caught_up = false;
+
// Load snapshot from leader, replacing the running StateMachine
std::string snapshot_path = reader->get_path();
snapshot_path.append(std::string("/") + db_snapshot_name);