Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Persist collection's next_seq_id.

Kishore Nallanb7654baa74e7
diff --git a/TODO.md b/TODO.md
index bf66be74..0db7c477 100644
--- a/TODO.md
+++ b/TODO.md
@@ -34,7 +34,7 @@
- Proper logging
- ~~Have set inside topster itself~~
- Restore records as well on restart (like for meta)
-- Persist next_seq_id
+- ~~Persist next_seq_id~~
- ~~collection_id should be int, not string~~
- clean special chars before indexing
diff --git a/include/collection.h b/include/collection.h
index 0d21f445..5468dccf 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -34,9 +34,12 @@ private:
const std::string SEQ_ID_PREFIX = "$SI";
const std::string DOC_ID_PREFIX = "$DI";
+ const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";
+
std::string get_seq_id_key(uint32_t seq_id);
std::string get_doc_id_key(std::string doc_id);
+ std::string get_collection_next_seq_id_key(std::string collection_name);
uint32_t get_next_seq_id();
static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index ba9915f1..dae016ab 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -20,16 +20,17 @@ private:
const std::string NEXT_COLLECTION_ID_KEY = "$CI";
const std::string COLLECTION_NAME_PREFIX = "$CN";
+ const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";
const std::string COLLECTION_NAME_KEY = "name";
const std::string COLLECTION_ID_KEY = "id";
- const std::string COLLECTION_NEXT_SEQ_ID_KEY = "next_seq_id";
const std::string COLLECTION_SEARCH_FIELDS_KEY = "search_fields";
const std::string COLLECTION_RANK_FIELDS_KEY = "rank_fields";
CollectionManager();
- std::string get_collection_name_key(std::string name);
+ std::string get_collection_name_key(std::string collection_name);
+ std::string get_collection_next_seq_id_key(std::string collection_name);
public:
static CollectionManager& get_instance() {
diff --git a/include/store.h b/include/store.h
index 62037d56..b092dac7 100644
--- a/include/store.h
+++ b/include/store.h
@@ -5,6 +5,24 @@
#include <string>
#include <rocksdb/db.h>
#include <rocksdb/options.h>
+#include <rocksdb/merge_operator.h>
+
+class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {
+public:
+ virtual bool Merge(const rocksdb::Slice& key, const rocksdb::Slice* existing_value, const rocksdb::Slice& value,
+ std::string* new_value, rocksdb::Logger* logger) const override {
+ uint64_t existing = 0;
+ if (existing_value) {
+ existing = (uint64_t) std::stoi(existing_value->ToString());
+ }
+ *new_value = std::to_string(existing + std::stoi(value.ToString()));
+ return true;
+ }
+
+ virtual const char* Name() const override {
+ return "UInt64AddOperator";
+ }
+};
/*
* Abstraction for underlying KV store (RocksDB)
@@ -29,6 +47,7 @@ public:
options.create_if_missing = true;
options.write_buffer_size = 4*1048576;
options.max_write_buffer_number = 2;
+ options.merge_operator.reset(new UInt64AddOperator);
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
@@ -69,6 +88,10 @@ public:
delete iter;
}
+ void increment(const std::string & key, uint32_t value) {
+ db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));
+ }
+
void print_memory_usage() {
std::string index_usage;
db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
diff --git a/src/collection.cpp b/src/collection.cpp
index 69b94549..75d997ad 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -28,7 +28,8 @@ Collection::~Collection() {
}
uint32_t Collection::get_next_seq_id() {
- return ++next_seq_id;
+ store->increment(get_collection_next_seq_id_key(name), 1);
+ return next_seq_id++;
}
std::string Collection::add(std::string json_str) {
@@ -536,6 +537,10 @@ void Collection::remove(std::string id) {
store->remove(get_seq_id_key(seq_id));
}
+std::string Collection::get_collection_next_seq_id_key(std::string collection_name) {
+ return COLLECTION_NEXT_SEQ_PREFIX + collection_name + "_SEQ";
+}
+
std::string Collection::get_seq_id_key(uint32_t seq_id) {
return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::to_string(seq_id);
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index dbebb816..35f86218 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -25,6 +25,7 @@ void CollectionManager::init(Store *store) {
for(auto collection_meta_json: collection_meta_jsons) {
nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);
+ std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();
std::vector<field> search_fields;
nlohmann::json fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];
@@ -32,10 +33,13 @@ void CollectionManager::init(Store *store) {
for (nlohmann::json::iterator it = fields_map.begin(); it != fields_map.end(); ++it) {
search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});
}
+
+ std::string collection_next_seq_id;
+ store->get(get_collection_next_seq_id_key(this_collection_name), collection_next_seq_id);
- Collection* collection = new Collection(collection_meta[COLLECTION_NAME_KEY].get<std::string>(),
+ Collection* collection = new Collection(this_collection_name,
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
- collection_meta[COLLECTION_NEXT_SEQ_ID_KEY].get<uint32_t>(),
+ (const uint32_t) std::stoi(collection_next_seq_id),
store,
search_fields,
collection_meta[COLLECTION_RANK_FIELDS_KEY].get<std::vector<std::string>>());
@@ -61,10 +65,11 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
collection_meta[COLLECTION_NAME_KEY] = name;
collection_meta[COLLECTION_ID_KEY] = next_collection_id;
- collection_meta[COLLECTION_NEXT_SEQ_ID_KEY] = 0;
collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = search_fields_json;
collection_meta[COLLECTION_RANK_FIELDS_KEY] = rank_fields;
+
store->insert(get_collection_name_key(name), collection_meta.dump());
+ store->insert(get_collection_next_seq_id_key(name), std::to_string(0));
Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, rank_fields);
@@ -78,6 +83,10 @@ std::string CollectionManager::get_collection_name_key(std::string collection_na
return COLLECTION_NAME_PREFIX + collection_name;
}
+std::string CollectionManager::get_collection_next_seq_id_key(std::string collection_name) {
+ return COLLECTION_NEXT_SEQ_PREFIX + collection_name + "_SEQ";
+}
+
Collection* CollectionManager::get_collection(std::string collection_name) {
if(collections.count(get_collection_name_key(collection_name)) != 0) {
return collections.at(get_collection_name_key(collection_name));
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 0322de6c..b648e88a 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -32,6 +32,10 @@ protected:
std::string json_line;
+ // dummy record for record id 0: to make the test record IDs to match with line numbers
+ json_line = "{\"points\":10,\"title\":\"z\"}";
+ collection->add(json_line);
+
while (std::getline(infile, json_line)) {
collection->add(json_line);
}