Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

API for fetching a document ID and listing all collections.

Kishore Nallan916aaf652629
diff --git a/TODO.md b/TODO.md
index 49bc1152..3cfead19 100644
--- a/TODO.md
+++ b/TODO.md
@@ -44,6 +44,9 @@
- ~~JSONP response~~
- ~~"error":"Not found." is sent when query has no hits~~
- ~~Fix API response codes~~
+- ~~List all collections~~
+- ~~Fetch an individual document~~
+- ~~ID field should be a string: must validate~~
- Number of records in collection
- Use rocksdb batch put for atomic insertion
- When prefix=true, use token_ranking_field for token ordering only for last word
@@ -86,9 +89,8 @@
- ~~index a single document~~
- ~~delete a document by ID~~
- ~~query an index~~
- - delete an index
- - bulk insert multiple documents
- - fetch a document by ID
+ - ~~Drop an index~~
+ - ~~fetch a document by ID~~
**Clustering**
diff --git a/include/api.h b/include/api.h
index 8597ca81..359df3f5 100644
--- a/include/api.h
+++ b/include/api.h
@@ -2,6 +2,8 @@
#include "http_server.h"
+void get_collections(http_req & req, http_res & res);
+
void post_create_collection(http_req & req, http_res & res);
void del_drop_collection(http_req & req, http_res & res);
@@ -10,4 +12,6 @@ void get_search(http_req & req, http_res & res);
void post_add_document(http_req & req, http_res & res);
+void get_fetch_document(http_req & req, http_res & res);
+
void del_remove_document(http_req & req, http_res & res);
\ No newline at end of file
diff --git a/include/collection.h b/include/collection.h
index 4485774c..16296986 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -120,6 +120,8 @@ public:
std::string get_seq_id_collection_prefix();
+ std::string get_name();
+
uint32_t get_collection_id();
uint32_t get_next_seq_id();
@@ -142,6 +144,8 @@ public:
const size_t per_page = 10, const size_t page = 1,
const token_ordering token_order = FREQUENCY, const bool prefix = false);
+ Option<nlohmann::json> get(const std::string & id);
+
Option<std::string> remove(const std::string & id);
void score_results(const std::vector<sort_field> & sort_fields, const int & token_rank, Topster<100> &topster,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index cc6b2e74..e67a56f8 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -49,7 +49,9 @@ public:
const std::vector<field> & sort_fields,
const std::string & token_ranking_field = "");
- Collection* get_collection(std::string collection_name);
+ Collection* get_collection(const std::string & collection_name);
+
+ std::vector<Collection*> get_collections();
Option<bool> drop_collection(std::string collection_name);
diff --git a/src/api.cpp b/src/api.cpp
index e1745985..adab54ea 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -6,6 +6,21 @@
#include "collection.h"
#include "collection_manager.h"
+void get_collections(http_req & req, http_res & res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::vector<Collection*> collections = collectionManager.get_collections();
+ nlohmann::json json_response;
+ json_response["data"] = nlohmann::json::array();
+
+ for(Collection* collection: collections) {
+ nlohmann::json collection_map;
+ collection_map["name"] = collection->get_name();
+ json_response["data"].push_back(collection_map);
+ }
+
+ res.send_200(json_response.dump());
+}
+
void post_create_collection(http_req & req, http_res & res) {
nlohmann::json req_json;
@@ -258,6 +273,24 @@ void post_add_document(http_req & req, http_res & res) {
}
}
+void get_fetch_document(http_req & req, http_res & res) {
+ std::string doc_id = req.params["id"];
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection* collection = collectionManager.get_collection(req.params["collection"]);
+ if(collection == nullptr) {
+ return res.send_404();
+ }
+
+ Option<nlohmann::json> doc_option = collection->get(doc_id);
+
+ if(!doc_option.ok()) {
+ res.send(doc_option.code(), doc_option.error());
+ } else {
+ res.send_200(doc_option.get().dump());
+ }
+}
+
void del_remove_document(http_req & req, http_res & res) {
std::string doc_id = req.params["id"];
diff --git a/src/collection.cpp b/src/collection.cpp
index 84d16fb9..3092562f 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -58,8 +58,12 @@ Option<std::string> Collection::add(const std::string & json_str) {
if(document.count("id") == 0) {
document["id"] = seq_id_str;
+ } else if(!document["id"].is_string()) {
+ return Option<std::string>(400, "Document's `id` field should be a string.");
}
+ std::string doc_id = document["id"];
+
const Option<uint32_t> & index_memory_op = index_in_memory(document, seq_id);
if(!index_memory_op.ok()) {
@@ -69,7 +73,6 @@ Option<std::string> Collection::add(const std::string & json_str) {
store->insert(get_seq_id_key(seq_id), document.dump());
store->insert(get_doc_id_key(document["id"]), seq_id_str);
- std::string doc_id = document["id"];
return Option<std::string>(doc_id);
}
@@ -1067,6 +1070,23 @@ void Collection::remove_and_shift_offset_index(sorted_array &offset_index, const
delete[] new_array;
}
+Option<nlohmann::json> Collection::get(const std::string & id) {
+ std::string seq_id_str;
+ StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
+
+ if(status == StoreStatus::NOT_FOUND) {
+ return Option<nlohmann::json>(404, "Could not find a document with id: " + id);
+ }
+
+ uint32_t seq_id = (uint32_t) std::stol(seq_id_str);
+
+ std::string parsed_document;
+ store->get(get_seq_id_key(seq_id), parsed_document);
+
+ nlohmann::json document = nlohmann::json::parse(parsed_document);
+ return Option<nlohmann::json>(document);
+}
+
Option<std::string> Collection::remove(const std::string & id) {
std::string seq_id_str;
StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
@@ -1202,6 +1222,10 @@ std::string Collection::get_doc_id_key(const std::string & doc_id) {
return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;
}
+std::string Collection::get_name() {
+ return name;
+}
+
uint32_t Collection::get_collection_id() {
return collection_id;
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 885935b2..dd593092 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -145,7 +145,7 @@ Collection* CollectionManager::create_collection(std::string name, const std::ve
return new_collection;
}
-Collection* CollectionManager::get_collection(std::string collection_name) {
+Collection* CollectionManager::get_collection(const std::string & collection_name) {
if(collections.count(Collection::get_meta_key(collection_name)) != 0) {
return collections.at(Collection::get_meta_key(collection_name));
}
@@ -153,6 +153,20 @@ Collection* CollectionManager::get_collection(std::string collection_name) {
return nullptr;
}
+std::vector<Collection*> CollectionManager::get_collections() {
+ std::vector<Collection*> collection_vec;
+ for(auto kv: collections) {
+ collection_vec.push_back(kv.second);
+ }
+
+ std::sort(std::begin(collection_vec), std::end(collection_vec),
+ [] (Collection* lhs, Collection* rhs) {
+ return lhs->get_collection_id() > rhs->get_collection_id();
+ });
+
+ return collection_vec;
+}
+
Option<bool> CollectionManager::drop_collection(std::string collection_name) {
Collection* collection = get_collection(collection_name);
if(collection == nullptr) {
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 733ce0bd..31f0308d 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -32,8 +32,10 @@ int main(int argc, char **argv) {
server.del("/collections/:collection", del_drop_collection, true);
// document management
+ server.get("/collections", get_collections, true);
server.post("/collections/:collection", post_add_document, true);
server.get("/collections/:collection/search", get_search, false);
+ server.get("/collections/:collection/:id", get_fetch_document, true);
server.del("/collections/:collection/:id", del_remove_document, true);
server.run();
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index b713c05d..e1a9be7f 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -87,6 +87,24 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
ASSERT_EQ("1", next_collection_id);
}
+TEST_F(CollectionManagerTest, GetAllCollections) {
+ std::vector<Collection*> collection_vec = collectionManager.get_collections();
+ ASSERT_EQ(1, collection_vec.size());
+ ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
+
+ // try creating one more collection
+ collectionManager.create_collection("collection2", search_fields, facet_fields,
+ sort_fields_index, "points");
+ collection_vec = collectionManager.get_collections();
+ ASSERT_EQ(2, collection_vec.size());
+
+ // most recently created collection first
+ ASSERT_STREQ("collection2", collection_vec[0]->get_name().c_str());
+ ASSERT_STREQ("collection1", collection_vec[1]->get_name().c_str());
+
+ collectionManager.drop_collection("collection2");
+}
+
TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
std::string json_line;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index c4d4296b..14159e97 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -61,6 +61,22 @@ protected:
}
};
+TEST_F(CollectionTest, RetrieveADocumentById) {
+ Option<nlohmann::json> doc_option = collection->get("1");
+ ASSERT_TRUE(doc_option.ok());
+ nlohmann::json doc = doc_option.get();
+ std::string id = doc["id"];
+
+ doc_option = collection->get("foo");
+ ASSERT_TRUE(doc_option.ok());
+ doc = doc_option.get();
+ id = doc["id"];
+ ASSERT_STREQ("foo", id.c_str());
+
+ doc_option = collection->get("baz");
+ ASSERT_FALSE(doc_option.ok());
+}
+
TEST_F(CollectionTest, ExactSearchShouldBeStable) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("the", query_fields, "", facets, sort_fields, 0, 10).get();
@@ -1020,6 +1036,35 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
collectionManager.drop_collection("empty_coll");
}
+TEST_F(CollectionTest, IdFieldShouldBeAString) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("name", field_types::STRING)};
+ facet_fields = {field("tags", field_types::STRING_ARRAY)};
+
+ std::vector<field> sort_fields_index = { field("age", "INT32"), field("average", "INT32") };
+ std::vector<sort_field> sort_fields = { sort_field("age", "DESC"), sort_field("average", "DESC") };
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", fields, facet_fields, sort_fields_index, "age");
+ }
+
+ nlohmann::json doc;
+ doc["id"] = 101010;
+ doc["name"] = "Jane";
+ doc["age"] = 25;
+ doc["average"] = 98;
+ doc["tags"] = nlohmann::json::array();
+ doc["tags"].push_back("tag1");
+
+ Option<std::string> inserted_id_op = coll1->add(doc.dump());
+ ASSERT_FALSE(inserted_id_op.ok());
+ ASSERT_STREQ("Document's `id` field should be a string.", inserted_id_op.error().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionTest, DeletionOfADocument) {
collectionManager.drop_collection("collection");
@@ -1090,4 +1135,6 @@ TEST_F(CollectionTest, DeletionOfADocument) {
}
delete it;
ASSERT_EQ(3, num_keys);
+
+ collectionManager.drop_collection("collection_for_del");
}
\ No newline at end of file