Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Typesense API skeleton.

Kishore Nallana82765277f7b
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6f5b23c3..98f7f14b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -86,4 +86,4 @@ endif()
target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(benchmark for pthread ${ROCKSDB_LIBS} ${STD_LIB})
-target_link_libraries(typesense_test pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB})
+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB})
diff --git a/include/api.h b/include/api.h
new file mode 100644
index 00000000..1f1692c7
--- /dev/null
+++ b/include/api.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "http_server.h"
+
+void get_search(http_req & req, http_res & res);
+
+void post_add_document(http_req & req, http_res & res);
+
+void del_remove_document(http_req & req, http_res & res);
\ No newline at end of file
diff --git a/include/collection_manager.h b/include/collection_manager.h
index a71dc611..700be3d7 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -27,14 +27,14 @@ private:
CollectionManager();
+ ~CollectionManager() = default;
+
public:
- static CollectionManager& get_instance() {
+ static CollectionManager & get_instance() {
static CollectionManager instance;
return instance;
}
- ~CollectionManager();
-
CollectionManager(CollectionManager const&) = delete;
void operator=(CollectionManager const&) = delete;
diff --git a/include/http_server.h b/include/http_server.h
index 7ce52523..e5a5ce57 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -1,3 +1,5 @@
+#pragma once
+
#define H2O_USE_LIBUV 0
extern "C" {
@@ -13,23 +15,53 @@ extern "C" {
#include "collection_manager.h"
struct http_res {
- uint32_t status;
+ uint32_t status_code;
+ std::string body;
+
+ void send_404() {
+ status_code = 404;
+ body = "{\"message\": \"Not Found\"}";
+ }
+
+ void send_500(const std::string & res_body) {
+ status_code = 404;
+ body = res_body;
+ }
+
+ void send_200(const std::string & res_body) {
+ status_code = 404;
+ body = res_body;
+ }
+
+ void send_201(const std::string & res_body) {
+ status_code = 404;
+ body = res_body;
+ }
+};
+
+struct http_req {
+ std::map<std::string, std::string> params;
std::string body;
};
+struct route_path {
+ std::string http_method;
+ std::vector<std::string> path_parts;
+ void (*handler)(http_req & req, http_res &);
+
+ inline bool operator< (const route_path& rhs) const {
+ return true;
+ }
+};
+
class HttpServer {
private:
static h2o_globalconf_t config;
static h2o_context_t ctx;
static h2o_accept_ctx_t accept_ctx;
- static std::map<std::string, int (*)(h2o_handler_t *, h2o_req_t *)> route_map;
+ static std::vector<route_path> routes;
h2o_hostconf_t *hostconf;
- const std::string data_dir;
-
- Store *store = nullptr;
- CollectionManager & collectionManager = CollectionManager::get_instance();
- Collection *collection;
static void on_accept(h2o_socket_t *listener, const char *err);
@@ -38,16 +70,20 @@ private:
h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,
int (*on_req)(h2o_handler_t *, h2o_req_t *));
+ static const char* get_status_reason(uint32_t status_code);
+
+ static std::map<std::string, std::string> parse_query(const std::string& query);
+
static int catch_all_handler(h2o_handler_t *self, h2o_req_t *req);
public:
- HttpServer(const std::string & data_dir);
+ HttpServer();
~HttpServer();
- void get(const std::string & path, int (*handler)(h2o_handler_t *, h2o_req_t *));
+ void get(const std::string & path, void (*handler)(http_req & req, http_res &));
- void post();
+ void post(const std::string & path, void (*handler)(http_req &, http_res &));
void put();
diff --git a/src/api.cpp b/src/api.cpp
new file mode 100644
index 00000000..4461cbfb
--- /dev/null
+++ b/src/api.cpp
@@ -0,0 +1,108 @@
+#include <regex>
+#include "api.h"
+#include "collection.h"
+#include "collection_manager.h"
+
+void get_search(http_req & req, http_res & res) {
+ const char *NUM_TYPOS = "num_typos";
+ const char *PREFIX = "prefix";
+ const char *TOKEN_ORDERING = "token_ordering";
+ const char *FILTERS = "filters";
+
+ if(req.params.count(NUM_TYPOS) == 0) {
+ req.params[NUM_TYPOS] = "2";
+ }
+
+ if(req.params.count(PREFIX) == 0) {
+ req.params[PREFIX] = "false";
+ }
+
+ if(req.params.count(TOKEN_ORDERING) == 0) {
+ req.params[TOKEN_ORDERING] = "FREQUENCY";
+ }
+
+ std::string filter_str = req.params.count(FILTERS) != 0 ? req.params[FILTERS] : "";
+ //std::cout << "filter_str: " << filter_str << std::endl;
+
+ token_ordering token_order = (req.params[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY;
+
+ //printf("Query: %s\n", req.params["q"].c_str());
+ auto begin = std::chrono::high_resolution_clock::now();
+
+ std::vector<std::string> search_fields = {"title"};
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection* collection = collectionManager.get_collection(req.params["collection"]);
+
+ if(collection == nullptr) {
+ return res.send_404();
+ }
+
+ nlohmann::json result = collection->search(req.params["q"], search_fields, filter_str, { },
+ {"points"}, std::stoi(req.params[NUM_TYPOS]), 100, token_order, false);
+ const std::string & json_str = result.dump();
+ //std::cout << "JSON:" << json_str << std::endl;
+ struct rusage r_usage;
+ getrusage(RUSAGE_SELF,&r_usage);
+
+ //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;
+ res.send_200(json_str);
+
+ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken: " << timeMillis << "us" << std::endl;
+}
+
+void post_add_document(http_req & req, http_res & res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection* collection = collectionManager.get_collection(req.params["collection"]);
+
+ if(collection == nullptr) {
+ return res.send_404();
+ }
+
+ Option<std::string> inserted_id_op = collection->add(req.body);
+
+ nlohmann::json json_response;
+ static h2o_generator_t generator = {NULL, NULL};
+
+ if(!inserted_id_op.ok()) {
+ json_response["message"] = inserted_id_op.error();
+ res.send_500(json_response.dump());
+
+ } else {
+ json_response["id"] = inserted_id_op.get();
+ res.send_201(json_response.dump());
+ }
+}
+
+/*
+int del_remove_document(h2o_handler_t *self, h2o_req_t *req) {
+ h2o_iovec_t query = req->query_at != SIZE_MAX ?
+ h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :
+ h2o_iovec_init(H2O_STRLIT(""));
+
+ std::string query_str(query.base, query.len);
+ std::map<std::string, std::string> req.params = parse_query(query_str);
+
+ std::string doc_id = req.params["id"];
+
+ auto begin = std::chrono::high_resolution_clock::now();
+ collection->remove(doc_id);
+ long long int time_micro = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::high_resolution_clock::now() - begin).count();
+ std::cout << "Time taken: " << time_micro << "us" << std::endl;
+
+ nlohmann::json json_response;
+ json_response["id"] = doc_id;
+ json_response["status"] = "SUCCESS";
+
+ static h2o_generator_t generator = {NULL, NULL};
+ req->res.status = 200;
+ req->res.reason = "OK";
+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
+ h2o_start_response(req, &generator);
+ h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);
+ h2o_send(req, &body, 1, 1);
+ return 0;
+}
+*/
\ No newline at end of file
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index ca6d86c5..387dfa4b 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -135,12 +135,6 @@ Collection* CollectionManager::get_collection(std::string collection_name) {
return nullptr;
}
-CollectionManager::~CollectionManager() {
- for(auto kv: collections) {
- drop_collection(kv.first);
- }
-}
-
bool CollectionManager::drop_collection(std::string collection_name) {
Collection* collection = get_collection(collection_name);
if(collection == nullptr) {
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 1c2253cb..1ed57a09 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -1,13 +1,16 @@
#include "http_server.h"
+#include "string_utils.h"
+#include <regex>
h2o_globalconf_t HttpServer::config;
h2o_context_t HttpServer::ctx;
h2o_accept_ctx_t HttpServer::accept_ctx;
-std::map<std::string, int (*)(h2o_handler_t *, h2o_req_t *)> HttpServer::route_map;
+std::vector<route_path> HttpServer::routes;
-HttpServer::HttpServer(const std::string & data_dir): data_dir(data_dir) {
+HttpServer::HttpServer() {
h2o_config_init(&config);
hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
+ register_handler(hostconf, "/", catch_all_handler);
}
void HttpServer::on_accept(h2o_socket_t *listener, const char *err) {
@@ -48,8 +51,6 @@ int HttpServer::create_listener(void) {
}
int HttpServer::run() {
- store = new Store(data_dir);
- collectionManager.init(store);
signal(SIGPIPE, SIG_IGN);
h2o_context_init(&ctx, h2o_evloop_create(), &config);
@@ -75,26 +76,123 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const cha
return pathconf;
}
+const char* HttpServer::get_status_reason(uint32_t status_code) {
+ switch(status_code) {
+ case 200: return "OK";
+ case 201: return "Created";
+ case 400: return "Bad Request";
+ case 404: return "Not Found";
+ case 500: return "Internal Server Error";
+ default: return "";
+ }
+}
+
+
+std::map<std::string, std::string> HttpServer::parse_query(const std::string& query) {
+ std::map<std::string, std::string> query_map;
+ std::regex pattern("([\\w+%]+)=([^&]*)");
+
+ auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern);
+ auto words_end = std::sregex_iterator();
+
+ for (std::sregex_iterator i = words_begin; i != words_end; i++) {
+ std::string key = (*i)[1].str();
+ std::string raw_value = (*i)[2].str();
+ std::string value = StringUtils::url_decode(raw_value);
+ if(query_map.count(value) == 0) {
+ query_map[key] = value;
+ } else {
+ query_map[key] = query_map[key] + "&&" + value;
+ }
+ }
+
+ return query_map;
+}
+
int HttpServer::catch_all_handler(h2o_handler_t *self, h2o_req_t *req) {
- std::string http_method = std::string(req->method.base, req->method.len);
- const std::string & route_key = http_method + "_" + std::string(req->path.base, req->path.len);
- if(route_map.count(route_key) == 0) {
- // send 404
- return 1;
+ const std::string & http_method = std::string(req->method.base, req->method.len);
+ const std::string & path = std::string(req->path.base, req->path.len);
+ h2o_generator_t generator = {NULL, NULL};
+
+ std::vector<std::string> path_with_query_parts;
+ StringUtils::split(path, path_with_query_parts, "?");
+ const std::string & path_without_query = path_with_query_parts[0];
+
+ std::vector<std::string> path_parts;
+ StringUtils::split(path_without_query, path_parts, "/");
+
+ h2o_iovec_t query = req->query_at != SIZE_MAX ?
+ h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :
+ h2o_iovec_init(H2O_STRLIT(""));
+
+ std::string query_str(query.base, query.len);
+ std::map<std::string, std::string> query_map = parse_query(query_str);
+ const std::string & req_body = std::string(req->entity.base, req->entity.len);
+
+ for(const route_path & rpath: routes) {
+ if(rpath.path_parts.size() != path_parts.size() || rpath.http_method != http_method) {
+ continue;
+ }
+
+ bool found = true;
+
+ for(size_t i = 0; i < rpath.path_parts.size(); i++) {
+ const std::string & rpart = rpath.path_parts[i];
+ const std::string & given_part = path_parts[i];
+ if(rpart != given_part && rpart[0] != ':') {
+ found = false;
+ goto check_next_route;
+ }
+ }
+
+ check_next_route:
+
+ if(found) {
+ // routes match - iterate and extract path params
+ for(size_t i = 0; i < rpath.path_parts.size(); i++) {
+ const std::string & path_part = rpath.path_parts[i];
+ if(path_part[0] == ':') {
+ query_map.emplace(path_part.substr(1), path_parts[i]);
+ }
+ }
+
+ http_req request = {query_map, req_body};
+ http_res response;
+ (rpath.handler)(request, response);
+
+ h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);
+ req->res.status = response.status_code;
+ req->res.reason = get_status_reason(response.status_code);
+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
+ h2o_start_response(req, &generator);
+ h2o_send(req, &body, 1, 1);
+
+ return 0;
+ }
}
- (route_map[route_key])(self, req);
+ h2o_iovec_t res_body = h2o_strdup(&req->pool, "{ \"message\": \"Not Found\"}", SIZE_MAX);
+ req->res.status = 404;
+ req->res.reason = get_status_reason(404);
+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
+ h2o_start_response(req, &generator);
+ h2o_send(req, &res_body, 1, 1);
return 0;
}
-void HttpServer::get(const std::string & path, int (*handler)(h2o_handler_t *, h2o_req_t *)) {
- route_map.emplace(std::string("GET") + "_" + path, handler);
- register_handler(hostconf, path.c_str(), catch_all_handler);
+void HttpServer::get(const std::string & path, void (*handler)(http_req &, http_res &)) {
+ std::vector<std::string> path_parts;
+ StringUtils::split(path, path_parts, "/");
+ route_path rpath = {"GET", path_parts, handler};
+ routes.push_back(rpath);
}
-void HttpServer::post() {
-
+void HttpServer::post(const std::string & path, void (*handler)(http_req &, http_res &)) {
+ std::vector<std::string> path_parts;
+ StringUtils::split(path, path_parts, "/");
+ route_path rpath = {"POST", path_parts, handler};
+ routes.push_back(rpath);
}
void HttpServer::put() {
@@ -106,5 +204,5 @@ void HttpServer::del() {
}
HttpServer::~HttpServer() {
- delete store;
+
}
\ No newline at end of file
diff --git a/src/main/main.cpp b/src/main/main.cpp
index 52b28058..636e8e78 100644
--- a/src/main/main.cpp
+++ b/src/main/main.cpp
@@ -39,8 +39,6 @@ int main(int argc, char* argv[]) {
std::cout << i << std::endl;
}
- return 0;
-
const std::string state_dir_path = "/tmp/typesense-data";
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index daa16848..dfff6842 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -1,5 +1,6 @@
-#include "http_server.h"
#include <cmdline.h>
+#include "http_server.h"
+#include "api.h"
int main(int argc, char **argv) {
cmdline::parser options;
@@ -8,17 +9,20 @@ int main(int argc, char **argv) {
options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8080);
options.parse_check(argc, argv);
- HttpServer server(options.get<std::string>("data-dir"));
- server.get("/foo", [](h2o_handler_t *self, h2o_req_t *req) {
- h2o_generator_t generator = {NULL, NULL};
- h2o_iovec_t body = h2o_strdup(&req->pool, "{\"foo\": 123}", SIZE_MAX);
- req->res.status = 200;
- req->res.reason = "OK";
- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
- h2o_start_response(req, &generator);
- h2o_send(req, &body, 1, 1);
+ Store store(options.get<std::string>("data-dir"));
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ collectionManager.init(&store);
+
+ HttpServer server;
+
+ server.get("/search/:collection", get_search);
+ server.post("/search/:collection", post_add_document);
+
+ /*server.get("/search/:collection", [](http_req & req, http_res & res) -> int {
+ res.status_code = 200;
+ res.body = "{\"collection\": \"" + req.params["collection"] + "\"}";
return 0;
- });
+ });*/
server.run();
return 0;