Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Defensive checks against JSON parse failures.
Kishore Nallan9c1a8a2364da
diff --git a/TODO.md b/TODO.mdindex 1ce38d3b..6fcb2190 100644--- a/TODO.md+++ b/TODO.md@@ -57,13 +57,14 @@- ~~only last token should be prefix searched~~- ~~Prefix-search strings should not be null terminated~~- ~~sort results by float field~~+- ~~json::parse must be wrapped in try catch~~- Collection Manager collections map should store plain collection name- init_collection of Collection manager should probably take seq_id as param+- https support- Validate before string to int conversion in the http api layer- When field of "id" but not string, what happens?- Typo in prefix search- node score should be int32, no longer uint16 like in document struct-- json::parse must be wrapped in try catch- test for num_documents- test for string filter comparison: title < "foo"- test for token ranking on float fielddiff --git a/include/collection_manager.h b/include/collection_manager.hindex d191d1e6..ea1a6b8e 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -42,12 +42,12 @@ public:CollectionManager(CollectionManager const&) = delete;void operator=(CollectionManager const&) = delete;- void init(Store *store, const std::string & auth_key);+ Option<bool> init(Store *store, const std::string & auth_key);// frees in-memory data structures when server is shutdown - helps us run a memory leak detecter properlyvoid dispose();- Collection* init_collection(const std::string & collection_meta_json);+ Option<Collection*> init_collection(const std::string & collection_meta_json);void add_to_collections(Collection* collection);diff --git a/include/replicator.h b/include/replicator.hindex 0b76564c..514368fb 100644--- a/include/replicator.h+++ b/include/replicator.h@@ -131,7 +131,16 @@ public:if(replication_event->type == "ADD_COLLECTION_META") {CollectionManager & collection_manager = CollectionManager::get_instance();- Collection* collection = collection_manager.init_collection(replication_event->value);+ Option<Collection*> collection_op = collection_manager.init_collection(replication_event->value);++ if(!collection_op.ok()) {+ std::cerr << "Failed to initialize collection. Error: " << collection_op.error() << std::endl;+ std::cerr << "Replication event value: " << replication_event->value << std::endl;+ delete replication_event;+ exit(1);+ }++ Collection* collection = collection_op.get();collection_manager.add_to_collections(collection);}diff --git a/src/api.cpp b/src/api.cppindex 319bfc56..6931a662 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -110,7 +110,8 @@ void post_create_collection(http_req & req, http_res & res) {"[{\"name\": \"<field_name>\", \"type\": \"<field_type>\"}]");}- if(sort_field_json["type"] != field_types::INT32 && sort_field_json["type"] != field_types::INT64) {+ const std::string & sort_field_type = sort_field_json["type"].get<std::string>();+ if(sort_field_type != field_types::INT32 && sort_field_type != field_types::INT64) {return res.send_400("Sort field `" + sort_field_json["name"].get<std::string>() + "` must be an integer.");}@@ -187,7 +188,7 @@ void get_search(http_req & req, http_res & res) {StringUtils::split(req.params[SEARCH_BY], search_fields, ",");std::vector<std::string> facet_fields;- StringUtils::split(req.params[FACET_BY], facet_fields, "&&");+ StringUtils::split(req.params[FACET_BY], facet_fields, ",");std::vector<sort_by> sort_fields;if(req.params.count(SORT_BY) != 0) {diff --git a/src/collection.cpp b/src/collection.cppindex cc138d09..a7646464 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -791,10 +791,17 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<sfor(size_t field_order_kv_index = start_result_index; field_order_kv_index <= end_result_index; field_order_kv_index++) {const auto & field_order_kv = field_order_kvs[field_order_kv_index];+ const std::string& seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);+std::string value;- const std::string &seq_id_key = get_seq_id_key((uint32_t) field_order_kv.second.key);store->get(seq_id_key, value);- nlohmann::json document = nlohmann::json::parse(value);++ nlohmann::json document;+ try {+ document = nlohmann::json::parse(value);+ } catch(...) {+ return Option<nlohmann::json>(500, "Error while parsing stored document.");+ }// highlight query words in the resultconst std::string & field_name = search_fields[search_fields.size() - field_order_kv.first];@@ -1258,7 +1265,13 @@ Option<nlohmann::json> Collection::get(const std::string & id) {std::string parsed_document;store->get(get_seq_id_key(seq_id), parsed_document);- nlohmann::json document = nlohmann::json::parse(parsed_document);+ nlohmann::json document;+ try {+ document = nlohmann::json::parse(parsed_document);+ } catch(...) {+ return Option<nlohmann::json>(500, "Error while parsing stored document.");+ }+return Option<nlohmann::json>(document);}@@ -1275,7 +1288,12 @@ Option<std::string> Collection::remove(const std::string & id, const bool removestd::string parsed_document;store->get(get_seq_id_key(seq_id), parsed_document);- nlohmann::json document = nlohmann::json::parse(parsed_document);+ nlohmann::json document;+ try {+ document = nlohmann::json::parse(parsed_document);+ } catch(...) {+ return Option<std::string>(500, "Error while parsing stored document.");+ }for(auto & name_field: search_schema) {// Go through all the field names and find the keys+values so that they can be removed from in-memory indexdiff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex b0f3d7a3..ddfa19a0 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -8,8 +8,15 @@ CollectionManager::CollectionManager() {}-Collection* CollectionManager::init_collection(const std::string & collection_meta_json) {- nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json);+Option<Collection*> CollectionManager::init_collection(const std::string & collection_meta_json) {+ nlohmann::json collection_meta;++ try {+ collection_meta = nlohmann::json::parse(collection_meta_json);+ } catch(...) {+ return Option<Collection*>(500, "Error while parsing collection meta.");+ }+std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();std::vector<field> search_fields;@@ -49,7 +56,7 @@ Collection* CollectionManager::init_collection(const std::string & collection_mecollection_sort_fields,token_ranking_field);- return collection;+ return Option<Collection*>(collection);}void CollectionManager::add_to_collections(Collection* collection) {@@ -57,7 +64,7 @@ void CollectionManager::add_to_collections(Collection* collection) {collection_id_names.emplace(collection->get_collection_id(), collection->get_name());}-void CollectionManager::init(Store *store, const std::string & auth_key) {+Option<bool> CollectionManager::init(Store *store, const std::string & auth_key) {this->store = store;this->auth_key = auth_key;@@ -73,7 +80,13 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);for(auto & collection_meta_json: collection_meta_jsons) {- Collection* collection = init_collection(collection_meta_json);+ Option<Collection*> collection_op = init_collection(collection_meta_json);++ if(!collection_op.ok()) {+ return Option<bool>(collection_op.code(), collection_op.error());+ }++ Collection* collection = collection_op.get();// Fetch records from the store and re-create memory indexstd::vector<std::string> documents;@@ -82,7 +95,17 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {const std::string doc_json_str = iter->value().ToString();- nlohmann::json document = nlohmann::json::parse(doc_json_str);++ nlohmann::json document;+ try {+ document = nlohmann::json::parse(doc_json_str);+ } catch(...) {+ delete iter;+ return Option<bool>(500,+ std::string("Error while parsing stored document from collection " + collection->get_name() ++ " with key: ") + iter->key().ToString());+ }+uint32_t seq_id = collection->doc_id_to_seq_id(document["id"]);collection->index_in_memory(document, seq_id);iter->Next();@@ -93,7 +116,7 @@ void CollectionManager::init(Store *store, const std::string & auth_key) {add_to_collections(collection);}- std::cout << "Finished restoring all collections from disk." << std::endl;+ return Option<bool>(true);}void CollectionManager::dispose() {diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex d5e5ed11..a47ca196 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -36,7 +36,14 @@ int main(int argc, char **argv) {Store store(options.get<std::string>("data-dir"));CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(&store, options.get<std::string>("api-auth-key"));+ Option<bool> init_op = collectionManager.init(&store, options.get<std::string>("api-auth-key"));++ if(init_op.ok()) {+ std::cout << "Finished restoring all collections from disk." << std::endl;+ } else {+ std::cerr << "Failed initializing collections from store..." << std::endl;+ std::cerr << init_op.error() << std::endl;+ }server = new HttpServer(options.get<std::string>("listen-address"),