Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Typesense API skeleton.
Kishore Nallana82765277f7b
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 6f5b23c3..98f7f14b 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -86,4 +86,4 @@ endif()target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})target_link_libraries(benchmark for pthread ${ROCKSDB_LIBS} ${STD_LIB})-target_link_libraries(typesense_test pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB})+target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB})diff --git a/include/api.h b/include/api.hnew file mode 100644index 00000000..1f1692c7--- /dev/null+++ b/include/api.h@@ -0,0 +1,9 @@+#pragma once++#include "http_server.h"++void get_search(http_req & req, http_res & res);++void post_add_document(http_req & req, http_res & res);++void del_remove_document(http_req & req, http_res & res);\ No newline at end of filediff --git a/include/collection_manager.h b/include/collection_manager.hindex a71dc611..700be3d7 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -27,14 +27,14 @@ private:CollectionManager();+ ~CollectionManager() = default;+public:- static CollectionManager& get_instance() {+ static CollectionManager & get_instance() {static CollectionManager instance;return instance;}- ~CollectionManager();-CollectionManager(CollectionManager const&) = delete;void operator=(CollectionManager const&) = delete;diff --git a/include/http_server.h b/include/http_server.hindex 7ce52523..e5a5ce57 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -1,3 +1,5 @@+#pragma once+#define H2O_USE_LIBUV 0extern "C" {@@ -13,23 +15,53 @@ extern "C" {#include "collection_manager.h"struct http_res {- uint32_t status;+ uint32_t status_code;+ std::string body;++ void send_404() {+ status_code = 404;+ body = "{\"message\": \"Not Found\"}";+ }++ void send_500(const std::string & res_body) {+ status_code = 404;+ body = res_body;+ }++ void send_200(const std::string & res_body) {+ status_code = 404;+ body = res_body;+ }++ void send_201(const std::string & res_body) {+ status_code = 404;+ body = res_body;+ }+};++struct http_req {+ std::map<std::string, std::string> params;std::string body;};+struct route_path {+ std::string http_method;+ std::vector<std::string> path_parts;+ void (*handler)(http_req & req, http_res &);++ inline bool operator< (const route_path& rhs) const {+ return true;+ }+};+class HttpServer {private:static h2o_globalconf_t config;static h2o_context_t ctx;static h2o_accept_ctx_t accept_ctx;- static std::map<std::string, int (*)(h2o_handler_t *, h2o_req_t *)> route_map;+ static std::vector<route_path> routes;h2o_hostconf_t *hostconf;- const std::string data_dir;-- Store *store = nullptr;- CollectionManager & collectionManager = CollectionManager::get_instance();- Collection *collection;static void on_accept(h2o_socket_t *listener, const char *err);@@ -38,16 +70,20 @@ private:h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,int (*on_req)(h2o_handler_t *, h2o_req_t *));+ static const char* get_status_reason(uint32_t status_code);++ static std::map<std::string, std::string> parse_query(const std::string& query);+static int catch_all_handler(h2o_handler_t *self, h2o_req_t *req);public:- HttpServer(const std::string & data_dir);+ HttpServer();~HttpServer();- void get(const std::string & path, int (*handler)(h2o_handler_t *, h2o_req_t *));+ void get(const std::string & path, void (*handler)(http_req & req, http_res &));- void post();+ void post(const std::string & path, void (*handler)(http_req &, http_res &));void put();diff --git a/src/api.cpp b/src/api.cppnew file mode 100644index 00000000..4461cbfb--- /dev/null+++ b/src/api.cpp@@ -0,0 +1,108 @@+#include <regex>+#include "api.h"+#include "collection.h"+#include "collection_manager.h"++void get_search(http_req & req, http_res & res) {+ const char *NUM_TYPOS = "num_typos";+ const char *PREFIX = "prefix";+ const char *TOKEN_ORDERING = "token_ordering";+ const char *FILTERS = "filters";++ if(req.params.count(NUM_TYPOS) == 0) {+ req.params[NUM_TYPOS] = "2";+ }++ if(req.params.count(PREFIX) == 0) {+ req.params[PREFIX] = "false";+ }++ if(req.params.count(TOKEN_ORDERING) == 0) {+ req.params[TOKEN_ORDERING] = "FREQUENCY";+ }++ std::string filter_str = req.params.count(FILTERS) != 0 ? req.params[FILTERS] : "";+ //std::cout << "filter_str: " << filter_str << std::endl;++ token_ordering token_order = (req.params[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY;++ //printf("Query: %s\n", req.params["q"].c_str());+ auto begin = std::chrono::high_resolution_clock::now();++ std::vector<std::string> search_fields = {"title"};++ CollectionManager & collectionManager = CollectionManager::get_instance();+ Collection* collection = collectionManager.get_collection(req.params["collection"]);++ if(collection == nullptr) {+ return res.send_404();+ }++ nlohmann::json result = collection->search(req.params["q"], search_fields, filter_str, { },+ {"points"}, std::stoi(req.params[NUM_TYPOS]), 100, token_order, false);+ const std::string & json_str = result.dump();+ //std::cout << "JSON:" << json_str << std::endl;+ struct rusage r_usage;+ getrusage(RUSAGE_SELF,&r_usage);++ //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;+ res.send_200(json_str);++ long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();+ std::cout << "Time taken: " << timeMillis << "us" << std::endl;+}++void post_add_document(http_req & req, http_res & res) {+ CollectionManager & collectionManager = CollectionManager::get_instance();+ Collection* collection = collectionManager.get_collection(req.params["collection"]);++ if(collection == nullptr) {+ return res.send_404();+ }++ Option<std::string> inserted_id_op = collection->add(req.body);++ nlohmann::json json_response;+ static h2o_generator_t generator = {NULL, NULL};++ if(!inserted_id_op.ok()) {+ json_response["message"] = inserted_id_op.error();+ res.send_500(json_response.dump());++ } else {+ json_response["id"] = inserted_id_op.get();+ res.send_201(json_response.dump());+ }+}++/*+int del_remove_document(h2o_handler_t *self, h2o_req_t *req) {+ h2o_iovec_t query = req->query_at != SIZE_MAX ?+ h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :+ h2o_iovec_init(H2O_STRLIT(""));++ std::string query_str(query.base, query.len);+ std::map<std::string, std::string> req.params = parse_query(query_str);++ std::string doc_id = req.params["id"];++ auto begin = std::chrono::high_resolution_clock::now();+ collection->remove(doc_id);+ long long int time_micro = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::high_resolution_clock::now() - begin).count();+ std::cout << "Time taken: " << time_micro << "us" << std::endl;++ nlohmann::json json_response;+ json_response["id"] = doc_id;+ json_response["status"] = "SUCCESS";++ static h2o_generator_t generator = {NULL, NULL};+ req->res.status = 200;+ req->res.reason = "OK";+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));+ h2o_start_response(req, &generator);+ h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);+ h2o_send(req, &body, 1, 1);+ return 0;+}+*/\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex ca6d86c5..387dfa4b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -135,12 +135,6 @@ Collection* CollectionManager::get_collection(std::string collection_name) {return nullptr;}-CollectionManager::~CollectionManager() {- for(auto kv: collections) {- drop_collection(kv.first);- }-}-bool CollectionManager::drop_collection(std::string collection_name) {Collection* collection = get_collection(collection_name);if(collection == nullptr) {diff --git a/src/http_server.cpp b/src/http_server.cppindex 1c2253cb..1ed57a09 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -1,13 +1,16 @@#include "http_server.h"+#include "string_utils.h"+#include <regex>h2o_globalconf_t HttpServer::config;h2o_context_t HttpServer::ctx;h2o_accept_ctx_t HttpServer::accept_ctx;-std::map<std::string, int (*)(h2o_handler_t *, h2o_req_t *)> HttpServer::route_map;+std::vector<route_path> HttpServer::routes;-HttpServer::HttpServer(const std::string & data_dir): data_dir(data_dir) {+HttpServer::HttpServer() {h2o_config_init(&config);hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);+ register_handler(hostconf, "/", catch_all_handler);}void HttpServer::on_accept(h2o_socket_t *listener, const char *err) {@@ -48,8 +51,6 @@ int HttpServer::create_listener(void) {}int HttpServer::run() {- store = new Store(data_dir);- collectionManager.init(store);signal(SIGPIPE, SIG_IGN);h2o_context_init(&ctx, h2o_evloop_create(), &config);@@ -75,26 +76,123 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const chareturn pathconf;}+const char* HttpServer::get_status_reason(uint32_t status_code) {+ switch(status_code) {+ case 200: return "OK";+ case 201: return "Created";+ case 400: return "Bad Request";+ case 404: return "Not Found";+ case 500: return "Internal Server Error";+ default: return "";+ }+}+++std::map<std::string, std::string> HttpServer::parse_query(const std::string& query) {+ std::map<std::string, std::string> query_map;+ std::regex pattern("([\\w+%]+)=([^&]*)");++ auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern);+ auto words_end = std::sregex_iterator();++ for (std::sregex_iterator i = words_begin; i != words_end; i++) {+ std::string key = (*i)[1].str();+ std::string raw_value = (*i)[2].str();+ std::string value = StringUtils::url_decode(raw_value);+ if(query_map.count(value) == 0) {+ query_map[key] = value;+ } else {+ query_map[key] = query_map[key] + "&&" + value;+ }+ }++ return query_map;+}+int HttpServer::catch_all_handler(h2o_handler_t *self, h2o_req_t *req) {- std::string http_method = std::string(req->method.base, req->method.len);- const std::string & route_key = http_method + "_" + std::string(req->path.base, req->path.len);- if(route_map.count(route_key) == 0) {- // send 404- return 1;+ const std::string & http_method = std::string(req->method.base, req->method.len);+ const std::string & path = std::string(req->path.base, req->path.len);+ h2o_generator_t generator = {NULL, NULL};++ std::vector<std::string> path_with_query_parts;+ StringUtils::split(path, path_with_query_parts, "?");+ const std::string & path_without_query = path_with_query_parts[0];++ std::vector<std::string> path_parts;+ StringUtils::split(path_without_query, path_parts, "/");++ h2o_iovec_t query = req->query_at != SIZE_MAX ?+ h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :+ h2o_iovec_init(H2O_STRLIT(""));++ std::string query_str(query.base, query.len);+ std::map<std::string, std::string> query_map = parse_query(query_str);+ const std::string & req_body = std::string(req->entity.base, req->entity.len);++ for(const route_path & rpath: routes) {+ if(rpath.path_parts.size() != path_parts.size() || rpath.http_method != http_method) {+ continue;+ }++ bool found = true;++ for(size_t i = 0; i < rpath.path_parts.size(); i++) {+ const std::string & rpart = rpath.path_parts[i];+ const std::string & given_part = path_parts[i];+ if(rpart != given_part && rpart[0] != ':') {+ found = false;+ goto check_next_route;+ }+ }++ check_next_route:++ if(found) {+ // routes match - iterate and extract path params+ for(size_t i = 0; i < rpath.path_parts.size(); i++) {+ const std::string & path_part = rpath.path_parts[i];+ if(path_part[0] == ':') {+ query_map.emplace(path_part.substr(1), path_parts[i]);+ }+ }++ http_req request = {query_map, req_body};+ http_res response;+ (rpath.handler)(request, response);++ h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);+ req->res.status = response.status_code;+ req->res.reason = get_status_reason(response.status_code);+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));+ h2o_start_response(req, &generator);+ h2o_send(req, &body, 1, 1);++ return 0;+ }}- (route_map[route_key])(self, req);+ h2o_iovec_t res_body = h2o_strdup(&req->pool, "{ \"message\": \"Not Found\"}", SIZE_MAX);+ req->res.status = 404;+ req->res.reason = get_status_reason(404);+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));+ h2o_start_response(req, &generator);+ h2o_send(req, &res_body, 1, 1);return 0;}-void HttpServer::get(const std::string & path, int (*handler)(h2o_handler_t *, h2o_req_t *)) {- route_map.emplace(std::string("GET") + "_" + path, handler);- register_handler(hostconf, path.c_str(), catch_all_handler);+void HttpServer::get(const std::string & path, void (*handler)(http_req &, http_res &)) {+ std::vector<std::string> path_parts;+ StringUtils::split(path, path_parts, "/");+ route_path rpath = {"GET", path_parts, handler};+ routes.push_back(rpath);}-void HttpServer::post() {-+void HttpServer::post(const std::string & path, void (*handler)(http_req &, http_res &)) {+ std::vector<std::string> path_parts;+ StringUtils::split(path, path_parts, "/");+ route_path rpath = {"POST", path_parts, handler};+ routes.push_back(rpath);}void HttpServer::put() {@@ -106,5 +204,5 @@ void HttpServer::del() {}HttpServer::~HttpServer() {- delete store;+}\ No newline at end of filediff --git a/src/main/main.cpp b/src/main/main.cppindex 52b28058..636e8e78 100644--- a/src/main/main.cpp+++ b/src/main/main.cpp@@ -39,8 +39,6 @@ int main(int argc, char* argv[]) {std::cout << i << std::endl;}- return 0;-const std::string state_dir_path = "/tmp/typesense-data";diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex daa16848..dfff6842 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -1,5 +1,6 @@-#include "http_server.h"#include <cmdline.h>+#include "http_server.h"+#include "api.h"int main(int argc, char **argv) {cmdline::parser options;@@ -8,17 +9,20 @@ int main(int argc, char **argv) {options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8080);options.parse_check(argc, argv);- HttpServer server(options.get<std::string>("data-dir"));- server.get("/foo", [](h2o_handler_t *self, h2o_req_t *req) {- h2o_generator_t generator = {NULL, NULL};- h2o_iovec_t body = h2o_strdup(&req->pool, "{\"foo\": 123}", SIZE_MAX);- req->res.status = 200;- req->res.reason = "OK";- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));- h2o_start_response(req, &generator);- h2o_send(req, &body, 1, 1);+ Store store(options.get<std::string>("data-dir"));+ CollectionManager & collectionManager = CollectionManager::get_instance();+ collectionManager.init(&store);++ HttpServer server;++ server.get("/search/:collection", get_search);+ server.post("/search/:collection", post_add_document);++ /*server.get("/search/:collection", [](http_req & req, http_res & res) -> int {+ res.status_code = 200;+ res.body = "{\"collection\": \"" + req.params["collection"] + "\"}";return 0;- });+ });*/server.run();return 0;