Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix ordering of sequence id rocksdb keys.
Kishore Nallan3ef10b5bb03a
diff --git a/include/collection.h b/include/collection.hindex 8ef7d100..a839e775 100644--- a/include/collection.h+++ b/include/collection.h@@ -11,12 +11,6 @@class Collection {private:- // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store- static constexpr const char* COLLECTION_META_PREFIX = "$CM";- static constexpr const char* DOC_ID_PREFIX = "$DI";- static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";- static constexpr const char* SEQ_ID_PREFIX = "$SI";-std::string name;uint32_t collection_id;@@ -79,7 +73,7 @@ public:static std::string get_meta_key(std::string collection_name);- std::string get_seq_id_prefix();+ std::string get_seq_id_collection_prefix();uint32_t get_collection_id();@@ -102,9 +96,15 @@ public:void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,const uint32_t *result_ids, const size_t result_size) const;+ void index_in_memory(const nlohmann::json &document, uint32_t seq_id);+enum {MAX_SEARCH_TOKENS = 20};enum {MAX_RESULTS = 100};- void index_in_memory(const nlohmann::json &document, uint32_t seq_id);+ // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store+ static constexpr const char* COLLECTION_META_PREFIX = "$CM";+ static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";+ static constexpr const char* SEQ_ID_PREFIX = "$SI";+ static constexpr const char* DOC_ID_PREFIX = "$DI";};diff --git a/include/collection_manager.h b/include/collection_manager.hindex 6f102abb..675cdf30 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -18,8 +18,6 @@ private:// Using a ID instead of a collection's name makes renaming possibleuint32_t next_collection_id;- static constexpr const char* COLLECTION_META_PREFIX = "$CM";-static constexpr const char* COLLECTION_NAME_KEY = "name";static constexpr const char* COLLECTION_ID_KEY = "id";static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "search_fields";@@ -27,8 +25,6 @@ private:CollectionManager();- static std::string get_collection_meta_key(std::string collection_name);-public:static CollectionManager& get_instance() {static CollectionManager instance;diff --git a/src/collection.cpp b/src/collection.cppindex 5d4e7de5..8dbcc373 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -606,18 +606,18 @@ void Collection::remove(std::string id) {}std::string Collection::get_next_seq_id_key(std::string collection_name) {- return COLLECTION_NEXT_SEQ_PREFIX + collection_name + "_SEQ";+ return std::string(COLLECTION_NEXT_SEQ_PREFIX) + "_" + collection_name;}std::string Collection::get_seq_id_key(uint32_t seq_id) {// We can't simply do std::to_string() because we want to preserve the byte order- union byteuint32_t {- char bytes[4];- uint32_t i;- };- byteuint32_t buint;- buint.i = seq_id;- return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::string(buint.bytes);+ unsigned char bytes[4];+ bytes[0] = (unsigned char) ((seq_id >> 24) & 0xFF);+ bytes[1] = (unsigned char) ((seq_id >> 16) & 0xFF);+ bytes[2] = (unsigned char) ((seq_id >> 8) & 0xFF);+ bytes[3] = (unsigned char) ((seq_id & 0xFF));++ return get_seq_id_collection_prefix() + "_" + std::string(bytes, bytes+4);}std::string Collection::get_doc_id_key(std::string doc_id) {@@ -647,6 +647,6 @@ std::string Collection::get_meta_key(std::string collection_name) {return COLLECTION_META_PREFIX + collection_name;}-std::string Collection::get_seq_id_prefix() {- return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX;+std::string Collection::get_seq_id_collection_prefix() {+ return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);}\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 1af06a21..03b24682 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -21,7 +21,7 @@ void CollectionManager::init(Store *store) {}std::vector<std::string> collection_meta_jsons;- store->scan_fill(COLLECTION_META_PREFIX, collection_meta_jsons);+ store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);for(auto collection_meta_json: collection_meta_jsons) {nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);@@ -50,8 +50,8 @@ void CollectionManager::init(Store *store) {// Fetch records from the store and re-create memory indexstd::vector<std::string> documents;- const std::string seq_id_prefix = collection->get_seq_id_prefix();- rocksdb::Iterator* iter = store->scan(collection->get_seq_id_prefix());+ const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {const std::string doc_json_str = iter->value().ToString();@@ -101,10 +101,6 @@ Collection* CollectionManager::create_collection(std::string name, const std::vereturn new_collection;}-std::string CollectionManager::get_collection_meta_key(std::string collection_name) {- return COLLECTION_META_PREFIX + collection_name;-}-Collection* CollectionManager::get_collection(std::string collection_name) {if(collections.count(Collection::get_meta_key(collection_name)) != 0) {return collections.at(Collection::get_meta_key(collection_name));diff --git a/src/main/server.cpp b/src/main/server.cppindex c43af9ab..9304d420 100644--- a/src/main/server.cpp+++ b/src/main/server.cpp@@ -220,11 +220,10 @@ int main(int argc, char **argv) {collection = collectionManager.get_collection("collection");if(collection == nullptr) {collection = collectionManager.create_collection("collection", search_fields, rank_fields);+ //index_documents(std::string(ROOT_DIR)+"test/documents.jsonl");+ index_documents(argv[1]);}- //index_documents(std::string(ROOT_DIR)+"test/documents.jsonl");- index_documents(argv[1]);-h2o_config_init(&config);h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);register_handler(hostconf, "/add", post_add_document);