Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Streaming import basics.
kishorenc4a173ee49d6c
diff --git a/include/http_data.h b/include/http_data.hindex 8986a50f..ed07bf52 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -20,6 +20,8 @@ struct http_res {std::string body;bool final;+ h2o_generator_t* generator = nullptr;+http_res(): status_code(501), content_type_header("application/json; charset=utf-8"), final(true) {}@@ -116,6 +118,11 @@ struct http_req {std::string http_method;uint64_t route_hash;std::map<std::string, std::string> params;++ bool streaming;+ std::string stream_state;++ size_t chunk_length;std::string body;std::string metadata;@@ -124,24 +131,28 @@ struct http_req {}http_req(h2o_req_t* _req, const std::string & http_method, uint64_t route_hash,- const std::map<std::string, std::string> & params, std::string body):+ const std::map<std::string, std::string> & params, const std::string& body):_req(_req), http_method(http_method), route_hash(route_hash), params(params),- body(body) {+ streaming(false), stream_state("NON_STREAMING"), body(body) {}// NOTE: we don't ser/de all fields, only ones needed for write forwarding+ // Take care to check for existence of key to ensure backward compatibility during upgradevoid deserialize(const std::string& serialized_content) {nlohmann::json content = nlohmann::json::parse(serialized_content);route_hash = content["route_hash"];body = content["body"];- metadata = content.count("metadata") != 0 ? content["metadata"] : "";for (nlohmann::json::iterator it = content["params"].begin(); it != content["params"].end(); ++it) {params.emplace(it.key(), it.value());}+ metadata = content.count("metadata") != 0 ? content["metadata"] : "";+ streaming = content.count("streaming") != 0 ? content["streaming"] : "";+ stream_state = content.count("stream_state") != 0 ? content["stream_state"] : "";+_req = nullptr;}@@ -149,6 +160,8 @@ struct http_req {nlohmann::json content;content["route_hash"] = route_hash;content["params"] = params;+ content["streaming"] = streaming;+ content["stream_state"] = stream_state;content["body"] = body;content["metadata"] = metadata;@@ -165,12 +178,14 @@ struct route_path {std::string http_method;std::vector<std::string> path_parts;bool (*handler)(http_req &, http_res &);+ bool async_req;bool async_res;std::string action;route_path(const std::string &httpMethod, const std::vector<std::string> &pathParts,- bool (*handler)(http_req &, http_res &), bool async) :- http_method(httpMethod), path_parts(pathParts), handler(handler), async_res(async) {+ bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) :+ http_method(httpMethod), path_parts(pathParts), handler(handler),+ async_req(async_req), async_res(async_res) {action = _get_action();}@@ -239,10 +254,9 @@ struct route_path {struct h2o_custom_generator_t {h2o_generator_t super;- bool (*handler)(http_req* req, http_res* res, void* data);+ bool (*req_handler)(http_req& req, http_res& res);http_req* req;http_res* res;- void* data;};struct h2o_custom_res_message_t {diff --git a/include/http_server.h b/include/http_server.hindex d26037e7..4c4322fa 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -29,6 +29,23 @@ struct h2o_custom_timer_t {}};+struct h2o_custom_req_handler_t {+ h2o_handler_t super;+ HttpServer* http_server;+};++struct async_req_ctx_t {+ http_req* request;+ http_res* response;+ h2o_custom_req_handler_t* handler;+ route_path *rpath;++ async_req_ctx_t(http_req *request, http_res* response, h2o_custom_req_handler_t *handler, route_path *rpath) :+ request(request), response(response), handler(handler), rpath(rpath) {++ }+};+class HttpServer {private:h2o_globalconf_t config;@@ -77,10 +94,16 @@ private:static std::map<std::string, std::string> parse_query(const std::string& query);- static int catch_all_handler(h2o_handler_t *self, h2o_req_t *req);+ static int catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req);++ static void response_proceed(h2o_generator_t *generator, h2o_req_t *req);++ static void on_res_generator_dispose(void *self);static int send_response(h2o_req_t *req, int status_code, const std::string & message);+ static int async_req_cb(void *ctx, h2o_iovec_t chunk, int is_end_stream);+public:HttpServer(const std::string & version,const std::string & listen_address, uint32_t listen_port,@@ -99,19 +122,19 @@ public:void set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const route_path & rpath,const std::string & auth_key));- void get(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);+ void get(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);- void post(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);+ void post(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);- void put(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);+ void put(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);- void del(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);+ void del(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);void on(const std::string & message, bool (*handler)(void*));void send_message(const std::string & type, void* data);- void send_response(http_req* request, const http_res* response);+ void send_response(http_req* request, http_res* response);uint64_t find_route(const std::vector<std::string> & path_parts, const std::string & http_method,route_path** found_rpath);@@ -130,4 +153,7 @@ public:static constexpr const char* AUTH_HEADER = "x-typesense-api-key";static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";++ static int process_request(http_req* request, http_res* response, route_path *rpath,+ const h2o_custom_req_handler_t *req_handler);};\ No newline at end of filediff --git a/include/index.h b/include/index.hindex 3c274ed1..0bfd1413 100644--- a/include/index.h+++ b/include/index.h@@ -129,8 +129,6 @@ private:void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;- size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);-Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,diff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.hindex 34f0cff4..4ccca45d 100644--- a/include/typesense_server_utils.h+++ b/include/typesense_server_utils.h@@ -18,7 +18,7 @@ void catch_interrupt(int sig);bool directory_exists(const std::string & dir_path);-void stream_response(bool (*handler)(http_req* req, http_res* res, void* data),+void stream_response(bool (*req_handler)(http_req* req, http_res* res, void* data),http_req & request, http_res & response, void* data);void init_cmdline_options(cmdline::parser & options, int argc, char **argv);diff --git a/src/core_api.cpp b/src/core_api.cppindex 92dde5f5..1948b358 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -574,6 +574,16 @@ bool post_add_document(http_req & req, http_res & res) {return true;}+bool collection_import_handler(http_req* request, http_res* response, void* data) {+ if(request->_req->proceed_req) {+ int is_end_stream = (response->final) ? 1: 0;+ size_t written = request->chunk_length;+ request->_req->proceed_req(request->_req, written, is_end_stream);+ }++ return true;+}+bool post_import_documents(http_req & req, http_res & res) {CollectionManager & collectionManager = CollectionManager::get_instance();Collection* collection = collectionManager.get_collection(req.params["collection"]);@@ -583,12 +593,35 @@ bool post_import_documents(http_req & req, http_res & res) {return false;}+ //LOG(INFO) << "req.body.size: " << req.body.size();+std::vector<std::string> json_lines;StringUtils::split(req.body, json_lines, "\n");+ if(req.stream_state == "END" || req.stream_state == "NON_STREAMING") {+ req.body = "";+ } else {+ if(!json_lines.empty()) {+ // check if req.body had incomplete last record+ if(!nlohmann::json::accept(json_lines.back())) {+ // eject partial record+ req.body = json_lines.back();+ json_lines.pop_back();+ }+ }+ }++ //LOG(INFO) << "json_lines.size: " << json_lines.size() << ", req.stream_state: " << req.stream_state;++ if(json_lines.empty() && !req.body.empty()) {+ // detects the case when only one partial record arrives as a chunk+ return true;+ }+const Option<nlohmann::json>& res_op = collection->add_many(json_lines);if(!res_op.ok()) {+ // FIXME: cannot set new status code midst of an ongoing import responseres.set(res_op.code(), res_op.error());return false;}@@ -608,6 +641,11 @@ bool post_import_documents(http_req & req, http_res & res) {res.content_type_header = "text/plain; charset=utf8";res.set_200(ss.str());++ if(req._req) {+ stream_response(collection_import_handler, req, res, nullptr);+ }+return true;}@@ -978,7 +1016,7 @@ bool async_write_request(void *data) {AsyncIndexArg* index_arg = static_cast<AsyncIndexArg*>(data);std::unique_ptr<AsyncIndexArg> index_arg_guard(index_arg);- bool async_call = false;+ bool async_res = false;if(index_arg->req->route_hash == static_cast<uint64_t>(ROUTE_CODES::NOT_FOUND)) {// route not found@@ -989,18 +1027,15 @@ bool async_write_request(void *data) {bool route_found = server->get_route(index_arg->req->route_hash, &found_rpath);if(route_found) {found_rpath->handler(*index_arg->req, *index_arg->res);- async_call = found_rpath->async_res;+ async_res = found_rpath->async_res;} else {index_arg->res->set_404();}}- if(!async_call && index_arg->req->_req != nullptr) {+ if(!async_res && index_arg->req->_req != nullptr) {// we have to return a response to the clientserver->send_response(index_arg->req, index_arg->res);- } else {- delete index_arg->req;- delete index_arg->res;}if(index_arg->promise != nullptr) {diff --git a/src/http_server.cpp b/src/http_server.cppindex a9eb97cf..798ef38b 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -10,11 +10,6 @@#include "raft_server.h"#include "logger.h"-struct h2o_custom_req_handler_t {- h2o_handler_t super;- HttpServer* http_server;-};-HttpServer::HttpServer(const std::string & version, const std::string & listen_address,uint32_t listen_port, const std::string & ssl_cert_path,const std::string & ssl_cert_key_path, bool cors_enabled):@@ -163,7 +158,7 @@ int HttpServer::run(ReplicationState* replication_state) {LOG(ERROR) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);return 1;} else {- LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port;+ LOG(INFO) << "Typesense has started listening on port " << listen_port;}message_dispatcher->on(STOP_SERVER_MESSAGE, HttpServer::on_stop_server);@@ -211,6 +206,9 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const chahandler->http_server = this;handler->super.on_req = on_req;+ // Enable streaming request body+ handler->super.supports_request_streaming = 1;+compress_args.min_size = 256; // don't gzip less than this sizecompress_args.brotli.quality = -1; // disable, not widely supportedcompress_args.gzip.quality = 1; // fastest@@ -270,8 +268,29 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, conreturn static_cast<uint64_t>(ROUTE_CODES::NOT_FOUND);}-int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {- h2o_custom_req_handler_t *self = (h2o_custom_req_handler_t *)_self;+void HttpServer::on_res_generator_dispose(void *self) {+ //LOG(INFO) << "on_res_generator_dispose fires";+ h2o_custom_generator_t* res_generator = static_cast<h2o_custom_generator_t*>(self);++ // res_generator itself is reference counted, so we only free members+ delete res_generator->req;+ delete res_generator->res;+}++void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {+ //LOG(INFO) << "response_proceed called";+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);+ custom_generator->req_handler(*custom_generator->req, *custom_generator->res);++ if(custom_generator->req->_req->proceed_req) {+ int is_end_stream = (custom_generator->res->final) ? 1: 0;+ size_t written = custom_generator->req->chunk_length;+ custom_generator->req->_req->proceed_req(custom_generator->req->_req, written, is_end_stream);+ }+}++int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {+ h2o_custom_req_handler_t *h2o_handler = (h2o_custom_req_handler_t *)_h2o_handler;const std::string & http_method = std::string(req->method.base, req->method.len);const std::string & path = std::string(req->path.base, req->path.len);@@ -281,7 +300,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {const std::string & path_without_query = path_with_query_parts[0];// Handle CORS- if(self->http_server->cors_enabled) {+ if(h2o_handler->http_server->cors_enabled) {h2o_add_header_by_str(&req->pool, &req->res.headers, H2O_STRLIT("access-control-allow-origin"),0, NULL, H2O_STRLIT("*"));@@ -318,7 +337,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {// Except for health check, wait for replicating state to be ready before allowing requests// Follower or leader must have started AND data must also have been loaded- if(path_without_query != "/health" && !self->http_server->get_replication_state()->is_ready()) {+ if(path_without_query != "/health" && !h2o_handler->http_server->get_replication_state()->is_ready()) {std::string message = "{ \"message\": \"Not Ready\"}";return send_response(req, 503, message);}@@ -345,7 +364,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {}route_path *rpath = nullptr;- uint64_t route_hash = self->http_server->find_route(path_parts, http_method, &rpath);+ uint64_t route_hash = h2o_handler->http_server->find_route(path_parts, http_method, &rpath);if(route_hash == static_cast<uint64_t>(ROUTE_CODES::NOT_FOUND)) {std::string message = "{ \"message\": \"Not Found\"}";@@ -360,18 +379,27 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {}}- bool authenticated = self->http_server->auth_handler(query_map, *rpath, api_auth_key_sent);+ bool authenticated = h2o_handler->http_server->auth_handler(query_map, *rpath, api_auth_key_sent);if(!authenticated) {std::string message = std::string("{\"message\": \"Forbidden - a valid `") + AUTH_HEADER +"` header must be sent.\"}";return send_response(req, 401, message);}- const std::string & req_body = std::string(req->entity.base, req->entity.len);-- http_req* request = new http_req(req, http_method, route_hash, query_map, req_body);+ const std::string & body = std::string(req->entity.base, req->entity.len);+ http_req* request = new http_req(req, rpath->http_method, route_hash, query_map, body);http_res* response = new http_res();+ // add custom generator with a dipose function for cleaning up resources+ h2o_custom_generator_t* custom_generator = static_cast<h2o_custom_generator_t*>(+ h2o_mem_alloc_shared(&req->pool, sizeof(*custom_generator), on_res_generator_dispose)+ );+ custom_generator->super = h2o_generator_t {response_proceed, nullptr};+ custom_generator->req = request;+ custom_generator->res = response;+ custom_generator->req_handler = rpath->handler;+ response->generator = &custom_generator->super;+// routes match and is an authenticated request// do any additional pre-request middleware operations hereif(rpath->action == "keys:create") {@@ -379,30 +407,100 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {request->metadata = StringUtils::randstring(AuthManager::KEY_LEN);}- // for writes, we defer to replication_state- if(http_method == "POST" || http_method == "PUT" || http_method == "DELETE") {- self->http_server->get_replication_state()->write(request, response);+ if(req->proceed_req == nullptr) {+ // Full request body is already available, so we don't care if handler is async or not+ return process_request(request, response, rpath, h2o_handler);+ } else {+ // Only partial request body is available.+ // If async_req is true, the request handler function will be invoked multiple times, for each chunk+ request->streaming = rpath->async_req;+ async_req_ctx_t* async_req_ctx = new async_req_ctx_t(request, response, h2o_handler, rpath);++ //LOG(INFO) << "Partial request body length: " << req->entity.len;++ req->write_req.cb = async_req_cb;+ req->write_req.ctx = async_req_ctx;+ int is_end_entity = 0;+ req->proceed_req(req, req->entity.len, is_end_entity);+ }++ return 0;+}++int HttpServer::async_req_cb(void *ctx, h2o_iovec_t chunk, int is_end_stream) {+ async_req_ctx_t* req_ctx = static_cast<async_req_ctx_t*>(ctx);++ http_req* request = req_ctx->request;+ http_res* response = req_ctx->response;++ LOG(INFO) << "chunk.len: " << chunk.len << ", is_end_stream: " << is_end_stream;++ request->chunk_length = chunk.len;++ std::string chunk_str(chunk.base, chunk.len);+ request->body += chunk_str;++ if(request->streaming || is_end_stream) {+ // Handler should be invoked for every chunk for streaming requests+ // For a non streaming request, buffer body and invoke only at the end++ // default value for response is NON_STREAMING+ // for streaming requests, we have to set: START, IN_PROGRESS or END+ if(request->streaming) {+ if(is_end_stream) {+ request->stream_state = "END";+ } else if(request->stream_state == "NON_STREAMING") {+ // `NON_STREAMING` default value indicates first chunk of streaming body+ request->stream_state = "START";+ } else {+ request->stream_state = "IN_PROGRESS";+ }+ }++ process_request(request, response, req_ctx->rpath, req_ctx->handler);+ }++ if(!is_end_stream && !request->streaming) {+ // streaming requests will call proceed_req in an async fashion+ // so we have to handle this only for non streaming+ size_t written = chunk.len;+ int stream_ended = 0;+ request->_req->proceed_req(request->_req, written, stream_ended);+ }++ if (is_end_stream) {+ // deletes only the container -- individual requests and response are deleted by response handler+ delete req_ctx;+ }++ return 0;+}++int HttpServer::process_request(http_req* request, http_res* response, route_path *rpath,+ const h2o_custom_req_handler_t *handler) {++ // for writes, we delegate to replication_state to handle response+ if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE") {+ handler->http_server->get_replication_state()->write(request, response);return 0;}+ // for reads, we will invoke the request handler and handle response as well+(rpath->handler)(*request, *response);if(!rpath->async_res) {// If a handler is marked as async res, it's responsible for sending the response itself in an async fashion- // otherwise, we send the response on behalf of the handler- self->http_server->send_response(request, response);+ // otherwise, we send the whole body response on behalf of the handler+ handler->http_server->send_response(request, response);}return 0;}-void HttpServer::send_message(const std::string & type, void* data) {- message_dispatcher->send_message(type, data);-}--void HttpServer::send_response(http_req* request, const http_res* response) {+void HttpServer::send_response(http_req* request, http_res* response) {h2o_req_t* req = request->_req;- h2o_generator_t generator = {NULL, NULL};+ h2o_generator_t& generator = *response->generator;h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);req->res.status = response->status_code;@@ -411,9 +509,6 @@ void HttpServer::send_response(http_req* request, const http_res* response) {nullptr, response->content_type_header.c_str(), response->content_type_header.size());h2o_start_response(req, &generator);h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);-- delete request;- delete response;}int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string & message) {@@ -427,36 +522,40 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::stringreturn 0;}+void HttpServer::send_message(const std::string & type, void* data) {+ message_dispatcher->send_message(type, data);+}+void HttpServer::set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const route_path& rpath,const std::string& auth_key)) {auth_handler = handler;}-void HttpServer::get(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {+void HttpServer::get(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");- route_path rpath("GET", path_parts, handler, async);+ route_path rpath("GET", path_parts, handler, async_req, async_res);routes.emplace_back(rpath.route_hash(), rpath);}-void HttpServer::post(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {+void HttpServer::post(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");- route_path rpath("POST", path_parts, handler, async);+ route_path rpath("POST", path_parts, handler, async_req, async_res);routes.emplace_back(rpath.route_hash(), rpath);}-void HttpServer::put(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {+void HttpServer::put(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");- route_path rpath("PUT", path_parts, handler, async);+ route_path rpath("PUT", path_parts, handler, async_req, async_res);routes.emplace_back(rpath.route_hash(), rpath);}-void HttpServer::del(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {+void HttpServer::del(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");- route_path rpath("DELETE", path_parts, handler, async);+ route_path rpath("DELETE", path_parts, handler, async_req, async_res);routes.emplace_back(rpath.route_hash(), rpath);}diff --git a/src/index.cpp b/src/index.cppindex e920c35e..95389432 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -784,8 +784,10 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si// every element in `query_suggestion` contains a token and its associated hitsstd::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);- /*for(size_t i=0; i < query_suggestion.size(); i++) {- LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key;+ /*LOG(INFO) << "n: " << n;+ for(size_t i=0; i < query_suggestion.size(); i++) {+ LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "+ << query_suggestion[i]->values->ids.getLength();}*/// initialize results with the starting element (for further intersection)@@ -857,32 +859,13 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, sisearched_queries.push_back(query_suggestion);+ //LOG(INFO) << "all_result_ids_len: " << all_result_ids_len << ", typo_tokens_threshold: " << typo_tokens_threshold;if(all_result_ids_len >= typo_tokens_threshold) {break;}}}-size_t Index::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,- uint32_t **results_out) {- uint32_t *results = nullptr;- size_t results_length = 0;-- uint32_t *prev_results = nullptr;- size_t prev_results_length = 0;-- for(const std::pair<uint32_t*, size_t> & result_array_pair: result_array_pairs) {- results_length = ArrayUtils::or_scalar(prev_results, prev_results_length, result_array_pair.first,- result_array_pair.second, &results);- delete [] prev_results;- prev_results = results;- prev_results_length = results_length;- }-- *results_out = results;- return results_length;-}-Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {uint32_t* filter_ids = nullptr;uint32_t filter_ids_length = 0;diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 788fde93..9e91b7d0 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -23,8 +23,8 @@ void master_server_routes() {server->post("/collections/:collection/documents", post_add_document);server->get("/collections/:collection/documents/search", get_search);- server->post("/collections/:collection/documents/import", post_import_documents);- server->get("/collections/:collection/documents/export", get_export_documents, true);+ server->post("/collections/:collection/documents/import", post_import_documents, true, true);+ server->get("/collections/:collection/documents/export", get_export_documents, false, true);server->get("/collections/:collection/documents/:id", get_fetch_document);server->del("/collections/:collection/documents/:id", del_remove_document);diff --git a/src/raft_server.cpp b/src/raft_server.cppindex b0986752..03f54dc9 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -161,16 +161,12 @@ void ReplicationState::write(http_req* request, http_res* response) {// To avoid ABA problemtask.expected_term = leader_term.load(butil::memory_order_relaxed);+ //LOG(INFO) << ":::" << "body size before apply: " << request->body.size();+// Now the task is applied to the group, waiting for the result.return node->apply(task);}-void ReplicationState::read(http_res* response) {- // NOT USED:- // For consistency, reads to followers could be rejected.- // Currently, we don't do implement reads via raft.-}-void ReplicationState::on_apply(braft::Iterator& iter) {// A batch of tasks are committed, which must be processed through// |iter|@@ -186,6 +182,8 @@ void ReplicationState::on_apply(braft::Iterator& iter) {ReplicationClosure* c = dynamic_cast<ReplicationClosure*>(iter.done());response = c->get_response();request = c->get_request();++ //LOG(INFO) << ":::" << "body size inside apply: " << request->body.size();} else {// Parse request from the logresponse = new http_res;@@ -217,6 +215,12 @@ void ReplicationState::on_apply(braft::Iterator& iter) {}}+void ReplicationState::read(http_res* response) {+ // NOT USED:+ // For consistency, reads to followers could be rejected.+ // Currently, we don't do implement reads via raft.+}+void* ReplicationState::save_snapshot(void* arg) {SnapshotArg* sa = static_cast<SnapshotArg*>(arg);std::unique_ptr<SnapshotArg> arg_guard(sa);diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 99af2e01..b7725320 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -68,49 +68,33 @@ Option<std::string> fetch_file_contents(const std::string & file_path) {return Option<std::string>(content);}-void response_proceed(h2o_generator_t *generator, h2o_req_t *req) {- h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);- custom_generator->handler(custom_generator->req, custom_generator->res, custom_generator->data);+void stream_response(bool (*req_handler)(http_req* req, http_res* res, void* data),+ http_req & request, http_res & response, void* data) {+ h2o_req_t* req = request._req;+ h2o_custom_generator_t *custom_generator = reinterpret_cast<h2o_custom_generator_t *>(response.generator);++ if (request.stream_state == "START" || request.stream_state == "NON_STREAMING") {+ req->res.status = response.status_code;+ req->res.reason = http_res::get_status_reason(response.status_code);+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL,+ response.content_type_header.c_str(),+ response.content_type_header.size());+ h2o_start_response(req, &custom_generator->super);+ }- h2o_iovec_t body = h2o_strdup(&req->pool, custom_generator->res->body.c_str(), SIZE_MAX);- const h2o_send_state_t state = custom_generator->res->final ?- H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;+ custom_generator->res->final = (request.stream_state == "END" || request.stream_state == "NON_STREAMING");++ h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);+ const h2o_send_state_t state = custom_generator->res->final ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;h2o_send(req, &body, 1, state);- if(custom_generator->res->final) {- h2o_dispose_request(req);+ if(state == H2O_SEND_STATE_FINAL) {delete custom_generator->req;delete custom_generator->res;delete custom_generator;}}-void response_stop(h2o_generator_t *generator, h2o_req_t *req) {- h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);-- h2o_dispose_request(req);- delete custom_generator->req;- delete custom_generator->res;- delete custom_generator;-}--void stream_response(bool (*handler)(http_req* req, http_res* res, void* data),- http_req & request, http_res & response, void* data) {- h2o_req_t* req = request._req;- h2o_custom_generator_t* custom_generator = new h2o_custom_generator_t {- h2o_generator_t {response_proceed, response_stop}, handler, &request, &response, data- };-- req->res.status = response.status_code;- req->res.reason = http_res::get_status_reason(response.status_code);- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL, response.content_type_header.c_str(),- response.content_type_header.size());- h2o_start_response(req, &custom_generator->super);-- h2o_iovec_t body = h2o_strdup(&req->pool, "", SIZE_MAX);- h2o_send(req, &body, 1, H2O_SEND_STATE_IN_PROGRESS);-}-void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.set_program_name("./typesense-server");diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cppindex 7582cf11..76316289 100644--- a/test/auth_manager_test.cpp+++ b/test/auth_manager_test.cpp@@ -163,12 +163,12 @@ TEST_F(AuthManagerTest, VerifyAuthentication) {}TEST_F(AuthManagerTest, GenerationOfAPIAction) {- route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false);- route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false);- route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false);- route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false);- route_path rpath_doc_delete = route_path("DELETE", {"collections", ":collection", "documents", ":id"}, nullptr, false);- route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false);+ route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false, false);+ route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false, false);+ route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false, false);+ route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false, false);+ route_path rpath_doc_delete = route_path("DELETE", {"collections", ":collection", "documents", ":id"}, nullptr, false, false);+ route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false, false);ASSERT_STREQ("documents:search", rpath_search._get_action().c_str());ASSERT_STREQ("collections:get", rpath_coll_get._get_action().c_str());diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 1cc44ff1..9ca0ea5a 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -77,4 +77,9 @@ TEST(StringUtilsTest, ShouldSplitString) {ASSERT_STREQ("{\"s\":\"Line one.\\nLine two.\\n\"}", lines[0].c_str());ASSERT_STREQ("{\"s\":\"Line 1.\\nLine 2.\\n\"}", lines[1].c_str());++ // empty string should produce empty list+ std::vector<std::string> lines_empty;+ StringUtils::split("", lines_empty, "\n");+ ASSERT_TRUE(lines_empty.empty());}