Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle scenarios when store get fails or doesn't find data when it should have.

Kishore Nallanb73dbebd2d81
diff --git a/TODO.md b/TODO.md
index dc413cb5..db053aa4 100644
--- a/TODO.md
+++ b/TODO.md
@@ -88,9 +88,12 @@
- ~~Add docs/explanation around ranking calc~~
- ~~UTF-8 normalization~~
- ~~Use rocksdb batch put for atomic insertion~~
+- ~~Proper logging~~
+- ~~Handle store-get() not finding a key~~
+- Deprecate union type punning
+- Deprecate converting integer to string verbatim
- NOT operator support
- > INT32_MAX validation for float field
-- Proper logging
- highlight of string arrays?
- test for token ranking on float field
- test for float int field deletion during doc deletion
@@ -99,7 +102,6 @@
- Query token ids should match query token ordering
- ID should not have "/"
- Group results by field
-- Handle store-get() not finding a key
- Delete using range: https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys
- Test for string utils
- Prevent string copy during indexing
diff --git a/include/collection.h b/include/collection.h
index 97d4c532..5a384f9f 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -80,7 +80,7 @@ public:
static uint32_t deserialize_seq_id_key(std::string serialized_seq_id);
- uint32_t doc_id_to_seq_id(std::string doc_id);
+ Option<uint32_t> doc_id_to_seq_id(std::string doc_id);
std::vector<std::string> get_facet_fields();
diff --git a/include/store.h b/include/store.h
index 65df539f..dff7ace9 100644
--- a/include/store.h
+++ b/include/store.h
@@ -32,7 +32,6 @@ public:
enum StoreStatus {
FOUND,
- OK,
NOT_FOUND,
ERROR
};
@@ -97,15 +96,16 @@ public:
StoreStatus get(const std::string& key, std::string& value) const {
rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
- if(status.IsNotFound()) {
- return StoreStatus::NOT_FOUND;
+ if(status.ok()) {
+ return StoreStatus::FOUND;
}
- if(!status.ok()) {
- return StoreStatus::ERROR;
+ if(status.IsNotFound()) {
+ return StoreStatus::NOT_FOUND;
}
- return StoreStatus::FOUND;
+ LOG(WARNING) << "Error while fetching the key: " << key << " - status is: " << status.ToString();
+ return StoreStatus::ERROR;
}
bool remove(const std::string& key) {
diff --git a/src/collection.cpp b/src/collection.cpp
index 4d6e3dfa..8b466e50 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -514,14 +514,19 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
const auto & field_order_kv = field_order_kvs[field_order_kv_index];
const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);
- std::string value;
- store->get(seq_id_key, value);
+ std::string json_doc_str;
+ StoreStatus json_doc_status = store->get(seq_id_key, json_doc_str);
+
+ if(json_doc_status != StoreStatus::FOUND) {
+ LOG(WARNING) << "Could not locate the JSON document for sequence ID: " << seq_id_key;
+ continue;
+ }
nlohmann::json wrapper_doc;
nlohmann::json document;
try {
- document = nlohmann::json::parse(value);
+ document = nlohmann::json::parse(json_doc_str);
} catch(...) {
return Option<nlohmann::json>(500, "Error while parsing stored document.");
}
@@ -639,16 +644,29 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
Option<nlohmann::json> Collection::get(const std::string & id) {
std::string seq_id_str;
- StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
+ StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);
- if(status == StoreStatus::NOT_FOUND) {
+ if(seq_id_status == StoreStatus::NOT_FOUND) {
return Option<nlohmann::json>(404, "Could not find a document with id: " + id);
}
+ if(seq_id_status == StoreStatus::ERROR) {
+ return Option<nlohmann::json>(500, "Error while fetching the document.");
+ }
+
uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
std::string parsed_document;
- store->get(get_seq_id_key(seq_id), parsed_document);
+ StoreStatus doc_status = store->get(get_seq_id_key(seq_id), parsed_document);
+
+ if(doc_status == StoreStatus::NOT_FOUND) {
+ LOG(WARNING) << "Sequence ID exists, but document is missing for id: " << id;
+ return Option<nlohmann::json>(404, "Could not find a document with id: " + id);
+ }
+
+ if(doc_status == StoreStatus::ERROR) {
+ return Option<nlohmann::json>(500, "Error while fetching the document.");
+ }
nlohmann::json document;
try {
@@ -662,16 +680,29 @@ Option<nlohmann::json> Collection::get(const std::string & id) {
Option<std::string> Collection::remove(const std::string & id, const bool remove_from_store) {
std::string seq_id_str;
- StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
+ StoreStatus seq_id_status = store->get(get_doc_id_key(id), seq_id_str);
- if(status == StoreStatus::NOT_FOUND) {
+ if(seq_id_status == StoreStatus::NOT_FOUND) {
return Option<std::string>(404, "Could not find a document with id: " + id);
}
+ if(seq_id_status == StoreStatus::ERROR) {
+ return Option<std::string>(500, "Error while fetching the document.");
+ }
+
uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
std::string parsed_document;
- store->get(get_seq_id_key(seq_id), parsed_document);
+ StoreStatus doc_status = store->get(get_seq_id_key(seq_id), parsed_document);
+
+ if(doc_status == StoreStatus::NOT_FOUND) {
+ LOG(WARNING) << "Sequence ID exists, but document is missing for id: " << id;
+ return Option<std::string>(404, "Could not find a document with id: " + id);
+ }
+
+ if(doc_status == StoreStatus::ERROR) {
+ return Option<std::string>(500, "Error while fetching the document.");
+ }
nlohmann::json document;
try {
@@ -731,11 +762,19 @@ uint32_t Collection::get_collection_id() {
return collection_id;
}
-uint32_t Collection::doc_id_to_seq_id(std::string doc_id) {
+Option<uint32_t> Collection::doc_id_to_seq_id(std::string doc_id) {
std::string seq_id_str;
- store->get(get_doc_id_key(doc_id), seq_id_str);
- uint32_t seq_id = (uint32_t) std::stoi(seq_id_str);
- return seq_id;
+ StoreStatus status = store->get(get_doc_id_key(doc_id), seq_id_str);
+ if(status == StoreStatus::FOUND) {
+ uint32_t seq_id = (uint32_t) std::stoi(seq_id_str);
+ return Option<uint32_t>(seq_id);
+ }
+
+ if(status == StoreStatus::NOT_FOUND) {
+ return Option<uint32_t>(404, "Not found.");
+ }
+
+ return Option<uint32_t>(500, "Error while fetching doc_id from store.");
}
std::vector<std::string> Collection::get_facet_fields() {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 6c52cb12..55e39589 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -43,8 +43,13 @@ Option<bool> CollectionManager::init(Store *store, const std::string & auth_key,
this->search_only_auth_key = search_only_auth_key;
std::string next_collection_id_str;
- store->get(NEXT_COLLECTION_ID_KEY, next_collection_id_str);
- if(!next_collection_id_str.empty()) {
+ StoreStatus next_coll_id_status = store->get(NEXT_COLLECTION_ID_KEY, next_collection_id_str);
+
+ if(next_coll_id_status == StoreStatus::ERROR) {
+ return Option<bool>(500, "Error while fetching the next collection id from the disk.");
+ }
+
+ if(next_coll_id_status == StoreStatus::FOUND) {
next_collection_id = (uint32_t) stoi(next_collection_id_str);
} else {
next_collection_id = 0;
@@ -64,8 +69,20 @@ Option<bool> CollectionManager::init(Store *store, const std::string & auth_key,
const std::string & this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();
std::string collection_next_seq_id_str;
- store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);
- uint32_t collection_next_seq_id = collection_next_seq_id_str.size() == 0 ? 0 :
+ StoreStatus next_seq_id_status = store->get(Collection::get_next_seq_id_key(this_collection_name),
+ collection_next_seq_id_str);
+
+ if(next_seq_id_status == StoreStatus::ERROR) {
+ return Option<bool>(500, "Error while fetching collection's next sequence ID from the disk for collection "
+ "`" + this_collection_name + "`");
+ }
+
+ if(next_seq_id_status == StoreStatus::NOT_FOUND && next_coll_id_status == StoreStatus::FOUND) {
+ return Option<bool>(500, "Next collection id was found, but collection's next sequence ID is missing for "
+ "`" + this_collection_name + "`");
+ }
+
+ uint32_t collection_next_seq_id = next_seq_id_status == StoreStatus::NOT_FOUND ? 0 :
(const uint32_t) std::stoi(collection_next_seq_id_str);
Collection* collection = init_collection(collection_meta, collection_next_seq_id);
@@ -83,13 +100,19 @@ Option<bool> CollectionManager::init(Store *store, const std::string & auth_key,
document = nlohmann::json::parse(doc_json_str);
} catch(...) {
delete iter;
- return Option<bool>(500,
- std::string("Error while parsing stored document from collection " + collection->get_name() +
- " with key: ") + iter->key().ToString());
+ return Option<bool>(500, std::string("Error while parsing stored document from collection " +
+ collection->get_name() + " with key: ") + iter->key().ToString());
+ }
+
+ Option<uint32_t> seq_id_op = collection->doc_id_to_seq_id(document["id"]);
+ if(!seq_id_op.ok()) {
+ delete iter;
+ return Option<bool>(500, std::string("Error while fetching sequence id of document id " +
+ document["id"].get<std::string>() + " in collection `" +
+ collection->get_name() + "`"));
}
- uint32_t seq_id = collection->doc_id_to_seq_id(document["id"]);
- collection->index_in_memory(document, seq_id);
+ collection->index_in_memory(document, seq_id_op.get());
iter->Next();
}