Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Persist collection's next_seq_id.
Kishore Nallanb7654baa74e7
diff --git a/TODO.md b/TODO.mdindex bf66be74..0db7c477 100644--- a/TODO.md+++ b/TODO.md@@ -34,7 +34,7 @@- Proper logging- ~~Have set inside topster itself~~- Restore records as well on restart (like for meta)-- Persist next_seq_id+- ~~Persist next_seq_id~~- ~~collection_id should be int, not string~~- clean special chars before indexingdiff --git a/include/collection.h b/include/collection.hindex 0d21f445..5468dccf 100644--- a/include/collection.h+++ b/include/collection.h@@ -34,9 +34,12 @@ private:const std::string SEQ_ID_PREFIX = "$SI";const std::string DOC_ID_PREFIX = "$DI";+ const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";+std::string get_seq_id_key(uint32_t seq_id);std::string get_doc_id_key(std::string doc_id);+ std::string get_collection_next_seq_id_key(std::string collection_name);uint32_t get_next_seq_id();static inline std::vector<art_leaf *> next_suggestion(const std::vector<std::vector<art_leaf *>> &token_leaves,diff --git a/include/collection_manager.h b/include/collection_manager.hindex ba9915f1..dae016ab 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -20,16 +20,17 @@ private:const std::string NEXT_COLLECTION_ID_KEY = "$CI";const std::string COLLECTION_NAME_PREFIX = "$CN";+ const std::string COLLECTION_NEXT_SEQ_PREFIX = "$CS";const std::string COLLECTION_NAME_KEY = "name";const std::string COLLECTION_ID_KEY = "id";- const std::string COLLECTION_NEXT_SEQ_ID_KEY = "next_seq_id";const std::string COLLECTION_SEARCH_FIELDS_KEY = "search_fields";const std::string COLLECTION_RANK_FIELDS_KEY = "rank_fields";CollectionManager();- std::string get_collection_name_key(std::string name);+ std::string get_collection_name_key(std::string collection_name);+ std::string get_collection_next_seq_id_key(std::string collection_name);public:static CollectionManager& get_instance() {diff --git a/include/store.h b/include/store.hindex 62037d56..b092dac7 100644--- a/include/store.h+++ b/include/store.h@@ -5,6 +5,24 @@#include <string>#include <rocksdb/db.h>#include <rocksdb/options.h>+#include <rocksdb/merge_operator.h>++class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {+public:+ virtual bool Merge(const rocksdb::Slice& key, const rocksdb::Slice* existing_value, const rocksdb::Slice& value,+ std::string* new_value, rocksdb::Logger* logger) const override {+ uint64_t existing = 0;+ if (existing_value) {+ existing = (uint64_t) std::stoi(existing_value->ToString());+ }+ *new_value = std::to_string(existing + std::stoi(value.ToString()));+ return true;+ }++ virtual const char* Name() const override {+ return "UInt64AddOperator";+ }+};/** Abstraction for underlying KV store (RocksDB)@@ -29,6 +47,7 @@ public:options.create_if_missing = true;options.write_buffer_size = 4*1048576;options.max_write_buffer_number = 2;+ options.merge_operator.reset(new UInt64AddOperator);// open DBrocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);@@ -69,6 +88,10 @@ public:delete iter;}+ void increment(const std::string & key, uint32_t value) {+ db->Merge(rocksdb::WriteOptions(), key, std::to_string(value));+ }+void print_memory_usage() {std::string index_usage;db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);diff --git a/src/collection.cpp b/src/collection.cppindex 69b94549..75d997ad 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -28,7 +28,8 @@ Collection::~Collection() {}uint32_t Collection::get_next_seq_id() {- return ++next_seq_id;+ store->increment(get_collection_next_seq_id_key(name), 1);+ return next_seq_id++;}std::string Collection::add(std::string json_str) {@@ -536,6 +537,10 @@ void Collection::remove(std::string id) {store->remove(get_seq_id_key(seq_id));}+std::string Collection::get_collection_next_seq_id_key(std::string collection_name) {+ return COLLECTION_NEXT_SEQ_PREFIX + collection_name + "_SEQ";+}+std::string Collection::get_seq_id_key(uint32_t seq_id) {return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::to_string(seq_id);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex dbebb816..35f86218 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -25,6 +25,7 @@ void CollectionManager::init(Store *store) {for(auto collection_meta_json: collection_meta_jsons) {nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);+ std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();std::vector<field> search_fields;nlohmann::json fields_map = collection_meta[COLLECTION_SEARCH_FIELDS_KEY];@@ -32,10 +33,13 @@ void CollectionManager::init(Store *store) {for (nlohmann::json::iterator it = fields_map.begin(); it != fields_map.end(); ++it) {search_fields.push_back({it.value()[fields::name], it.value()[fields::type]});}++ std::string collection_next_seq_id;+ store->get(get_collection_next_seq_id_key(this_collection_name), collection_next_seq_id);- Collection* collection = new Collection(collection_meta[COLLECTION_NAME_KEY].get<std::string>(),+ Collection* collection = new Collection(this_collection_name,collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),- collection_meta[COLLECTION_NEXT_SEQ_ID_KEY].get<uint32_t>(),+ (const uint32_t) std::stoi(collection_next_seq_id),store,search_fields,collection_meta[COLLECTION_RANK_FIELDS_KEY].get<std::vector<std::string>>());@@ -61,10 +65,11 @@ Collection* CollectionManager::create_collection(std::string name, const std::vecollection_meta[COLLECTION_NAME_KEY] = name;collection_meta[COLLECTION_ID_KEY] = next_collection_id;- collection_meta[COLLECTION_NEXT_SEQ_ID_KEY] = 0;collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = search_fields_json;collection_meta[COLLECTION_RANK_FIELDS_KEY] = rank_fields;+store->insert(get_collection_name_key(name), collection_meta.dump());+ store->insert(get_collection_next_seq_id_key(name), std::to_string(0));Collection* new_collection = new Collection(name, next_collection_id, 0, store, search_fields, rank_fields);@@ -78,6 +83,10 @@ std::string CollectionManager::get_collection_name_key(std::string collection_nareturn COLLECTION_NAME_PREFIX + collection_name;}+std::string CollectionManager::get_collection_next_seq_id_key(std::string collection_name) {+ return COLLECTION_NEXT_SEQ_PREFIX + collection_name + "_SEQ";+}+Collection* CollectionManager::get_collection(std::string collection_name) {if(collections.count(get_collection_name_key(collection_name)) != 0) {return collections.at(get_collection_name_key(collection_name));diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 0322de6c..b648e88a 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -32,6 +32,10 @@ protected:std::string json_line;+ // dummy record for record id 0: to make the test record IDs to match with line numbers+ json_line = "{\"points\":10,\"title\":\"z\"}";+ collection->add(json_line);+while (std::getline(infile, json_line)) {collection->add(json_line);}