Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix ordering of sequence id rocksdb keys.

Kishore Nallan3ef10b5bb03a
diff --git a/include/collection.h b/include/collection.h
index 8ef7d100..a839e775 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -11,12 +11,6 @@
class Collection {
private:
- // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
- static constexpr const char* COLLECTION_META_PREFIX = "$CM";
- static constexpr const char* DOC_ID_PREFIX = "$DI";
- static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
- static constexpr const char* SEQ_ID_PREFIX = "$SI";
-
std::string name;
uint32_t collection_id;
@@ -79,7 +73,7 @@ public:
static std::string get_meta_key(std::string collection_name);
- std::string get_seq_id_prefix();
+ std::string get_seq_id_collection_prefix();
uint32_t get_collection_id();
@@ -102,9 +96,15 @@ public:
void score_results(Topster<100> &topster, const int & token_rank, const std::vector<art_leaf *> &query_suggestion,
const uint32_t *result_ids, const size_t result_size) const;
+ void index_in_memory(const nlohmann::json &document, uint32_t seq_id);
+
enum {MAX_SEARCH_TOKENS = 20};
enum {MAX_RESULTS = 100};
- void index_in_memory(const nlohmann::json &document, uint32_t seq_id);
+ // Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
+ static constexpr const char* COLLECTION_META_PREFIX = "$CM";
+ static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
+ static constexpr const char* SEQ_ID_PREFIX = "$SI";
+ static constexpr const char* DOC_ID_PREFIX = "$DI";
};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 6f102abb..675cdf30 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -18,8 +18,6 @@ private:
// Using a ID instead of a collection's name makes renaming possible
uint32_t next_collection_id;
- static constexpr const char* COLLECTION_META_PREFIX = "$CM";
-
static constexpr const char* COLLECTION_NAME_KEY = "name";
static constexpr const char* COLLECTION_ID_KEY = "id";
static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "search_fields";
@@ -27,8 +25,6 @@ private:
CollectionManager();
- static std::string get_collection_meta_key(std::string collection_name);
-
public:
static CollectionManager& get_instance() {
static CollectionManager instance;
diff --git a/src/collection.cpp b/src/collection.cpp
index 5d4e7de5..8dbcc373 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -606,18 +606,18 @@ void Collection::remove(std::string id) {
}
std::string Collection::get_next_seq_id_key(std::string collection_name) {
- return COLLECTION_NEXT_SEQ_PREFIX + collection_name + "_SEQ";
+ return std::string(COLLECTION_NEXT_SEQ_PREFIX) + "_" + collection_name;
}
std::string Collection::get_seq_id_key(uint32_t seq_id) {
// We can't simply do std::to_string() because we want to preserve the byte order
- union byteuint32_t {
- char bytes[4];
- uint32_t i;
- };
- byteuint32_t buint;
- buint.i = seq_id;
- return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX + std::string(buint.bytes);
+ unsigned char bytes[4];
+ bytes[0] = (unsigned char) ((seq_id >> 24) & 0xFF);
+ bytes[1] = (unsigned char) ((seq_id >> 16) & 0xFF);
+ bytes[2] = (unsigned char) ((seq_id >> 8) & 0xFF);
+ bytes[3] = (unsigned char) ((seq_id & 0xFF));
+
+ return get_seq_id_collection_prefix() + "_" + std::string(bytes, bytes+4);
}
std::string Collection::get_doc_id_key(std::string doc_id) {
@@ -647,6 +647,6 @@ std::string Collection::get_meta_key(std::string collection_name) {
return COLLECTION_META_PREFIX + collection_name;
}
-std::string Collection::get_seq_id_prefix() {
- return std::to_string(collection_id) + "_" + SEQ_ID_PREFIX;
+std::string Collection::get_seq_id_collection_prefix() {
+ return std::to_string(collection_id) + "_" + std::string(SEQ_ID_PREFIX);
}
\ No newline at end of file
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 1af06a21..03b24682 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -21,7 +21,7 @@ void CollectionManager::init(Store *store) {
}
std::vector<std::string> collection_meta_jsons;
- store->scan_fill(COLLECTION_META_PREFIX, collection_meta_jsons);
+ store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);
for(auto collection_meta_json: collection_meta_jsons) {
nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);
@@ -50,8 +50,8 @@ void CollectionManager::init(Store *store) {
// Fetch records from the store and re-create memory index
std::vector<std::string> documents;
- const std::string seq_id_prefix = collection->get_seq_id_prefix();
- rocksdb::Iterator* iter = store->scan(collection->get_seq_id_prefix());
+ const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();
+ rocksdb::Iterator* iter = store->scan(seq_id_prefix);
while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
const std::string doc_json_str = iter->value().ToString();
@@ -101,10 +101,6 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
return new_collection;
}
-std::string CollectionManager::get_collection_meta_key(std::string collection_name) {
- return COLLECTION_META_PREFIX + collection_name;
-}
-
Collection* CollectionManager::get_collection(std::string collection_name) {
if(collections.count(Collection::get_meta_key(collection_name)) != 0) {
return collections.at(Collection::get_meta_key(collection_name));
diff --git a/src/main/server.cpp b/src/main/server.cpp
index c43af9ab..9304d420 100644
--- a/src/main/server.cpp
+++ b/src/main/server.cpp
@@ -220,11 +220,10 @@ int main(int argc, char **argv) {
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
collection = collectionManager.create_collection("collection", search_fields, rank_fields);
+ //index_documents(std::string(ROOT_DIR)+"test/documents.jsonl");
+ index_documents(argv[1]);
}
- //index_documents(std::string(ROOT_DIR)+"test/documents.jsonl");
- index_documents(argv[1]);
-
h2o_config_init(&config);
h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
register_handler(hostconf, "/add", post_add_document);