Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
API for fetching a document ID and listing all collections.
Kishore Nallan916aaf652629
diff --git a/TODO.md b/TODO.mdindex 49bc1152..3cfead19 100644--- a/TODO.md+++ b/TODO.md@@ -44,6 +44,9 @@- ~~JSONP response~~- ~~"error":"Not found." is sent when query has no hits~~- ~~Fix API response codes~~+- ~~List all collections~~+- ~~Fetch an individual document~~+- ~~ID field should be a string: must validate~~- Number of records in collection- Use rocksdb batch put for atomic insertion- When prefix=true, use token_ranking_field for token ordering only for last word@@ -86,9 +89,8 @@- ~~index a single document~~- ~~delete a document by ID~~- ~~query an index~~- - delete an index- - bulk insert multiple documents- - fetch a document by ID+ - ~~Drop an index~~+ - ~~fetch a document by ID~~**Clustering**diff --git a/include/api.h b/include/api.hindex 8597ca81..359df3f5 100644--- a/include/api.h+++ b/include/api.h@@ -2,6 +2,8 @@#include "http_server.h"+void get_collections(http_req & req, http_res & res);+void post_create_collection(http_req & req, http_res & res);void del_drop_collection(http_req & req, http_res & res);@@ -10,4 +12,6 @@ void get_search(http_req & req, http_res & res);void post_add_document(http_req & req, http_res & res);+void get_fetch_document(http_req & req, http_res & res);+void del_remove_document(http_req & req, http_res & res);\ No newline at end of filediff --git a/include/collection.h b/include/collection.hindex 4485774c..16296986 100644--- a/include/collection.h+++ b/include/collection.h@@ -120,6 +120,8 @@ public:std::string get_seq_id_collection_prefix();+ std::string get_name();+uint32_t get_collection_id();uint32_t get_next_seq_id();@@ -142,6 +144,8 @@ public:const size_t per_page = 10, const size_t page = 1,const token_ordering token_order = FREQUENCY, const bool prefix = false);+ Option<nlohmann::json> get(const std::string & id);+Option<std::string> remove(const std::string & id);void score_results(const std::vector<sort_field> & sort_fields, const int & token_rank, Topster<100> &topster,diff --git a/include/collection_manager.h b/include/collection_manager.hindex cc6b2e74..e67a56f8 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -49,7 +49,9 @@ public:const std::vector<field> & sort_fields,const std::string & token_ranking_field = "");- Collection* get_collection(std::string collection_name);+ Collection* get_collection(const std::string & collection_name);++ std::vector<Collection*> get_collections();Option<bool> drop_collection(std::string collection_name);diff --git a/src/api.cpp b/src/api.cppindex e1745985..adab54ea 100644--- a/src/api.cpp+++ b/src/api.cpp@@ -6,6 +6,21 @@#include "collection.h"#include "collection_manager.h"+void get_collections(http_req & req, http_res & res) {+ CollectionManager & collectionManager = CollectionManager::get_instance();+ std::vector<Collection*> collections = collectionManager.get_collections();+ nlohmann::json json_response;+ json_response["data"] = nlohmann::json::array();++ for(Collection* collection: collections) {+ nlohmann::json collection_map;+ collection_map["name"] = collection->get_name();+ json_response["data"].push_back(collection_map);+ }++ res.send_200(json_response.dump());+}+void post_create_collection(http_req & req, http_res & res) {nlohmann::json req_json;@@ -258,6 +273,24 @@ void post_add_document(http_req & req, http_res & res) {}}+void get_fetch_document(http_req & req, http_res & res) {+ std::string doc_id = req.params["id"];++ CollectionManager & collectionManager = CollectionManager::get_instance();+ Collection* collection = collectionManager.get_collection(req.params["collection"]);+ if(collection == nullptr) {+ return res.send_404();+ }++ Option<nlohmann::json> doc_option = collection->get(doc_id);++ if(!doc_option.ok()) {+ res.send(doc_option.code(), doc_option.error());+ } else {+ res.send_200(doc_option.get().dump());+ }+}+void del_remove_document(http_req & req, http_res & res) {std::string doc_id = req.params["id"];diff --git a/src/collection.cpp b/src/collection.cppindex 84d16fb9..3092562f 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -58,8 +58,12 @@ Option<std::string> Collection::add(const std::string & json_str) {if(document.count("id") == 0) {document["id"] = seq_id_str;+ } else if(!document["id"].is_string()) {+ return Option<std::string>(400, "Document's `id` field should be a string.");}+ std::string doc_id = document["id"];+const Option<uint32_t> & index_memory_op = index_in_memory(document, seq_id);if(!index_memory_op.ok()) {@@ -69,7 +73,6 @@ Option<std::string> Collection::add(const std::string & json_str) {store->insert(get_seq_id_key(seq_id), document.dump());store->insert(get_doc_id_key(document["id"]), seq_id_str);- std::string doc_id = document["id"];return Option<std::string>(doc_id);}@@ -1067,6 +1070,23 @@ void Collection::remove_and_shift_offset_index(sorted_array &offset_index, constdelete[] new_array;}+Option<nlohmann::json> Collection::get(const std::string & id) {+ std::string seq_id_str;+ StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);++ if(status == StoreStatus::NOT_FOUND) {+ return Option<nlohmann::json>(404, "Could not find a document with id: " + id);+ }++ uint32_t seq_id = (uint32_t) std::stol(seq_id_str);++ std::string parsed_document;+ store->get(get_seq_id_key(seq_id), parsed_document);++ nlohmann::json document = nlohmann::json::parse(parsed_document);+ return Option<nlohmann::json>(document);+}+Option<std::string> Collection::remove(const std::string & id) {std::string seq_id_str;StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);@@ -1202,6 +1222,10 @@ std::string Collection::get_doc_id_key(const std::string & doc_id) {return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;}+std::string Collection::get_name() {+ return name;+}+uint32_t Collection::get_collection_id() {return collection_id;}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 885935b2..dd593092 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -145,7 +145,7 @@ Collection* CollectionManager::create_collection(std::string name, const std::vereturn new_collection;}-Collection* CollectionManager::get_collection(std::string collection_name) {+Collection* CollectionManager::get_collection(const std::string & collection_name) {if(collections.count(Collection::get_meta_key(collection_name)) != 0) {return collections.at(Collection::get_meta_key(collection_name));}@@ -153,6 +153,20 @@ Collection* CollectionManager::get_collection(std::string collection_name) {return nullptr;}+std::vector<Collection*> CollectionManager::get_collections() {+ std::vector<Collection*> collection_vec;+ for(auto kv: collections) {+ collection_vec.push_back(kv.second);+ }++ std::sort(std::begin(collection_vec), std::end(collection_vec),+ [] (Collection* lhs, Collection* rhs) {+ return lhs->get_collection_id() > rhs->get_collection_id();+ });++ return collection_vec;+}+Option<bool> CollectionManager::drop_collection(std::string collection_name) {Collection* collection = get_collection(collection_name);if(collection == nullptr) {diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 733ce0bd..31f0308d 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -32,8 +32,10 @@ int main(int argc, char **argv) {server.del("/collections/:collection", del_drop_collection, true);// document management+ server.get("/collections", get_collections, true);server.post("/collections/:collection", post_add_document, true);server.get("/collections/:collection/search", get_search, false);+ server.get("/collections/:collection/:id", get_fetch_document, true);server.del("/collections/:collection/:id", del_remove_document, true);server.run();diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex b713c05d..e1a9be7f 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -87,6 +87,24 @@ TEST_F(CollectionManagerTest, CollectionCreation) {ASSERT_EQ("1", next_collection_id);}+TEST_F(CollectionManagerTest, GetAllCollections) {+ std::vector<Collection*> collection_vec = collectionManager.get_collections();+ ASSERT_EQ(1, collection_vec.size());+ ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());++ // try creating one more collection+ collectionManager.create_collection("collection2", search_fields, facet_fields,+ sort_fields_index, "points");+ collection_vec = collectionManager.get_collections();+ ASSERT_EQ(2, collection_vec.size());++ // most recently created collection first+ ASSERT_STREQ("collection2", collection_vec[0]->get_name().c_str());+ ASSERT_STREQ("collection1", collection_vec[1]->get_name().c_str());++ collectionManager.drop_collection("collection2");+}+TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");std::string json_line;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex c4d4296b..14159e97 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -61,6 +61,22 @@ protected:}};+TEST_F(CollectionTest, RetrieveADocumentById) {+ Option<nlohmann::json> doc_option = collection->get("1");+ ASSERT_TRUE(doc_option.ok());+ nlohmann::json doc = doc_option.get();+ std::string id = doc["id"];++ doc_option = collection->get("foo");+ ASSERT_TRUE(doc_option.ok());+ doc = doc_option.get();+ id = doc["id"];+ ASSERT_STREQ("foo", id.c_str());++ doc_option = collection->get("baz");+ ASSERT_FALSE(doc_option.ok());+}+TEST_F(CollectionTest, ExactSearchShouldBeStable) {std::vector<std::string> facets;nlohmann::json results = collection->search("the", query_fields, "", facets, sort_fields, 0, 10).get();@@ -1020,6 +1036,35 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {collectionManager.drop_collection("empty_coll");}+TEST_F(CollectionTest, IdFieldShouldBeAString) {+ Collection *coll1;++ std::vector<field> fields = {field("name", field_types::STRING)};+ facet_fields = {field("tags", field_types::STRING_ARRAY)};++ std::vector<field> sort_fields_index = { field("age", "INT32"), field("average", "INT32") };+ std::vector<sort_field> sort_fields = { sort_field("age", "DESC"), sort_field("average", "DESC") };++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", fields, facet_fields, sort_fields_index, "age");+ }++ nlohmann::json doc;+ doc["id"] = 101010;+ doc["name"] = "Jane";+ doc["age"] = 25;+ doc["average"] = 98;+ doc["tags"] = nlohmann::json::array();+ doc["tags"].push_back("tag1");++ Option<std::string> inserted_id_op = coll1->add(doc.dump());+ ASSERT_FALSE(inserted_id_op.ok());+ ASSERT_STREQ("Document's `id` field should be a string.", inserted_id_op.error().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionTest, DeletionOfADocument) {collectionManager.drop_collection("collection");@@ -1090,4 +1135,6 @@ TEST_F(CollectionTest, DeletionOfADocument) {}delete it;ASSERT_EQ(3, num_keys);++ collectionManager.drop_collection("collection_for_del");}\ No newline at end of file