Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Streaming import basics.

kishorenc4a173ee49d6c
diff --git a/include/http_data.h b/include/http_data.h
index 8986a50f..ed07bf52 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -20,6 +20,8 @@ struct http_res {
std::string body;
bool final;
+ h2o_generator_t* generator = nullptr;
+
http_res(): status_code(501), content_type_header("application/json; charset=utf-8"), final(true) {
}
@@ -116,6 +118,11 @@ struct http_req {
std::string http_method;
uint64_t route_hash;
std::map<std::string, std::string> params;
+
+ bool streaming;
+ std::string stream_state;
+
+ size_t chunk_length;
std::string body;
std::string metadata;
@@ -124,24 +131,28 @@ struct http_req {
}
http_req(h2o_req_t* _req, const std::string & http_method, uint64_t route_hash,
- const std::map<std::string, std::string> & params, std::string body):
+ const std::map<std::string, std::string> & params, const std::string& body):
_req(_req), http_method(http_method), route_hash(route_hash), params(params),
- body(body) {
+ streaming(false), stream_state("NON_STREAMING"), body(body) {
}
// NOTE: we don't ser/de all fields, only ones needed for write forwarding
+ // Take care to check for existence of key to ensure backward compatibility during upgrade
void deserialize(const std::string& serialized_content) {
nlohmann::json content = nlohmann::json::parse(serialized_content);
route_hash = content["route_hash"];
body = content["body"];
- metadata = content.count("metadata") != 0 ? content["metadata"] : "";
for (nlohmann::json::iterator it = content["params"].begin(); it != content["params"].end(); ++it) {
params.emplace(it.key(), it.value());
}
+ metadata = content.count("metadata") != 0 ? content["metadata"] : "";
+ streaming = content.count("streaming") != 0 ? content["streaming"] : "";
+ stream_state = content.count("stream_state") != 0 ? content["stream_state"] : "";
+
_req = nullptr;
}
@@ -149,6 +160,8 @@ struct http_req {
nlohmann::json content;
content["route_hash"] = route_hash;
content["params"] = params;
+ content["streaming"] = streaming;
+ content["stream_state"] = stream_state;
content["body"] = body;
content["metadata"] = metadata;
@@ -165,12 +178,14 @@ struct route_path {
std::string http_method;
std::vector<std::string> path_parts;
bool (*handler)(http_req &, http_res &);
+ bool async_req;
bool async_res;
std::string action;
route_path(const std::string &httpMethod, const std::vector<std::string> &pathParts,
- bool (*handler)(http_req &, http_res &), bool async) :
- http_method(httpMethod), path_parts(pathParts), handler(handler), async_res(async) {
+ bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) :
+ http_method(httpMethod), path_parts(pathParts), handler(handler),
+ async_req(async_req), async_res(async_res) {
action = _get_action();
}
@@ -239,10 +254,9 @@ struct route_path {
struct h2o_custom_generator_t {
h2o_generator_t super;
- bool (*handler)(http_req* req, http_res* res, void* data);
+ bool (*req_handler)(http_req& req, http_res& res);
http_req* req;
http_res* res;
- void* data;
};
struct h2o_custom_res_message_t {
diff --git a/include/http_server.h b/include/http_server.h
index d26037e7..4c4322fa 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -29,6 +29,23 @@ struct h2o_custom_timer_t {
}
};
+struct h2o_custom_req_handler_t {
+ h2o_handler_t super;
+ HttpServer* http_server;
+};
+
+struct async_req_ctx_t {
+ http_req* request;
+ http_res* response;
+ h2o_custom_req_handler_t* handler;
+ route_path *rpath;
+
+ async_req_ctx_t(http_req *request, http_res* response, h2o_custom_req_handler_t *handler, route_path *rpath) :
+ request(request), response(response), handler(handler), rpath(rpath) {
+
+ }
+};
+
class HttpServer {
private:
h2o_globalconf_t config;
@@ -77,10 +94,16 @@ private:
static std::map<std::string, std::string> parse_query(const std::string& query);
- static int catch_all_handler(h2o_handler_t *self, h2o_req_t *req);
+ static int catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req);
+
+ static void response_proceed(h2o_generator_t *generator, h2o_req_t *req);
+
+ static void on_res_generator_dispose(void *self);
static int send_response(h2o_req_t *req, int status_code, const std::string & message);
+ static int async_req_cb(void *ctx, h2o_iovec_t chunk, int is_end_stream);
+
public:
HttpServer(const std::string & version,
const std::string & listen_address, uint32_t listen_port,
@@ -99,19 +122,19 @@ public:
void set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const route_path & rpath,
const std::string & auth_key));
- void get(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);
+ void get(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);
- void post(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);
+ void post(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);
- void put(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);
+ void put(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);
- void del(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async = false);
+ void del(const std::string & path, bool (*handler)(http_req & req, http_res & res), bool async_req=false, bool async_res=false);
void on(const std::string & message, bool (*handler)(void*));
void send_message(const std::string & type, void* data);
- void send_response(http_req* request, const http_res* response);
+ void send_response(http_req* request, http_res* response);
uint64_t find_route(const std::vector<std::string> & path_parts, const std::string & http_method,
route_path** found_rpath);
@@ -130,4 +153,7 @@ public:
static constexpr const char* AUTH_HEADER = "x-typesense-api-key";
static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";
+
+ static int process_request(http_req* request, http_res* response, route_path *rpath,
+ const h2o_custom_req_handler_t *req_handler);
};
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index 3c274ed1..0bfd1413 100644
--- a/include/index.h
+++ b/include/index.h
@@ -129,8 +129,6 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
- size_t union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs, uint32_t **results_out);
-
Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
diff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.h
index 34f0cff4..4ccca45d 100644
--- a/include/typesense_server_utils.h
+++ b/include/typesense_server_utils.h
@@ -18,7 +18,7 @@ void catch_interrupt(int sig);
bool directory_exists(const std::string & dir_path);
-void stream_response(bool (*handler)(http_req* req, http_res* res, void* data),
+void stream_response(bool (*req_handler)(http_req* req, http_res* res, void* data),
http_req & request, http_res & response, void* data);
void init_cmdline_options(cmdline::parser & options, int argc, char **argv);
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 92dde5f5..1948b358 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -574,6 +574,16 @@ bool post_add_document(http_req & req, http_res & res) {
return true;
}
+bool collection_import_handler(http_req* request, http_res* response, void* data) {
+ if(request->_req->proceed_req) {
+ int is_end_stream = (response->final) ? 1: 0;
+ size_t written = request->chunk_length;
+ request->_req->proceed_req(request->_req, written, is_end_stream);
+ }
+
+ return true;
+}
+
bool post_import_documents(http_req & req, http_res & res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
Collection* collection = collectionManager.get_collection(req.params["collection"]);
@@ -583,12 +593,35 @@ bool post_import_documents(http_req & req, http_res & res) {
return false;
}
+ //LOG(INFO) << "req.body.size: " << req.body.size();
+
std::vector<std::string> json_lines;
StringUtils::split(req.body, json_lines, "\n");
+ if(req.stream_state == "END" || req.stream_state == "NON_STREAMING") {
+ req.body = "";
+ } else {
+ if(!json_lines.empty()) {
+ // check if req.body had incomplete last record
+ if(!nlohmann::json::accept(json_lines.back())) {
+ // eject partial record
+ req.body = json_lines.back();
+ json_lines.pop_back();
+ }
+ }
+ }
+
+ //LOG(INFO) << "json_lines.size: " << json_lines.size() << ", req.stream_state: " << req.stream_state;
+
+ if(json_lines.empty() && !req.body.empty()) {
+ // detects the case when only one partial record arrives as a chunk
+ return true;
+ }
+
const Option<nlohmann::json>& res_op = collection->add_many(json_lines);
if(!res_op.ok()) {
+ // FIXME: cannot set new status code midst of an ongoing import response
res.set(res_op.code(), res_op.error());
return false;
}
@@ -608,6 +641,11 @@ bool post_import_documents(http_req & req, http_res & res) {
res.content_type_header = "text/plain; charset=utf8";
res.set_200(ss.str());
+
+ if(req._req) {
+ stream_response(collection_import_handler, req, res, nullptr);
+ }
+
return true;
}
@@ -978,7 +1016,7 @@ bool async_write_request(void *data) {
AsyncIndexArg* index_arg = static_cast<AsyncIndexArg*>(data);
std::unique_ptr<AsyncIndexArg> index_arg_guard(index_arg);
- bool async_call = false;
+ bool async_res = false;
if(index_arg->req->route_hash == static_cast<uint64_t>(ROUTE_CODES::NOT_FOUND)) {
// route not found
@@ -989,18 +1027,15 @@ bool async_write_request(void *data) {
bool route_found = server->get_route(index_arg->req->route_hash, &found_rpath);
if(route_found) {
found_rpath->handler(*index_arg->req, *index_arg->res);
- async_call = found_rpath->async_res;
+ async_res = found_rpath->async_res;
} else {
index_arg->res->set_404();
}
}
- if(!async_call && index_arg->req->_req != nullptr) {
+ if(!async_res && index_arg->req->_req != nullptr) {
// we have to return a response to the client
server->send_response(index_arg->req, index_arg->res);
- } else {
- delete index_arg->req;
- delete index_arg->res;
}
if(index_arg->promise != nullptr) {
diff --git a/src/http_server.cpp b/src/http_server.cpp
index a9eb97cf..798ef38b 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -10,11 +10,6 @@
#include "raft_server.h"
#include "logger.h"
-struct h2o_custom_req_handler_t {
- h2o_handler_t super;
- HttpServer* http_server;
-};
-
HttpServer::HttpServer(const std::string & version, const std::string & listen_address,
uint32_t listen_port, const std::string & ssl_cert_path,
const std::string & ssl_cert_key_path, bool cors_enabled):
@@ -163,7 +158,7 @@ int HttpServer::run(ReplicationState* replication_state) {
LOG(ERROR) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);
return 1;
} else {
- LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port;
+ LOG(INFO) << "Typesense has started listening on port " << listen_port;
}
message_dispatcher->on(STOP_SERVER_MESSAGE, HttpServer::on_stop_server);
@@ -211,6 +206,9 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const cha
handler->http_server = this;
handler->super.on_req = on_req;
+ // Enable streaming request body
+ handler->super.supports_request_streaming = 1;
+
compress_args.min_size = 256; // don't gzip less than this size
compress_args.brotli.quality = -1; // disable, not widely supported
compress_args.gzip.quality = 1; // fastest
@@ -270,8 +268,29 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, con
return static_cast<uint64_t>(ROUTE_CODES::NOT_FOUND);
}
-int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
- h2o_custom_req_handler_t *self = (h2o_custom_req_handler_t *)_self;
+void HttpServer::on_res_generator_dispose(void *self) {
+ //LOG(INFO) << "on_res_generator_dispose fires";
+ h2o_custom_generator_t* res_generator = static_cast<h2o_custom_generator_t*>(self);
+
+ // res_generator itself is reference counted, so we only free members
+ delete res_generator->req;
+ delete res_generator->res;
+}
+
+void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
+ //LOG(INFO) << "response_proceed called";
+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);
+ custom_generator->req_handler(*custom_generator->req, *custom_generator->res);
+
+ if(custom_generator->req->_req->proceed_req) {
+ int is_end_stream = (custom_generator->res->final) ? 1: 0;
+ size_t written = custom_generator->req->chunk_length;
+ custom_generator->req->_req->proceed_req(custom_generator->req->_req, written, is_end_stream);
+ }
+}
+
+int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
+ h2o_custom_req_handler_t *h2o_handler = (h2o_custom_req_handler_t *)_h2o_handler;
const std::string & http_method = std::string(req->method.base, req->method.len);
const std::string & path = std::string(req->path.base, req->path.len);
@@ -281,7 +300,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
const std::string & path_without_query = path_with_query_parts[0];
// Handle CORS
- if(self->http_server->cors_enabled) {
+ if(h2o_handler->http_server->cors_enabled) {
h2o_add_header_by_str(&req->pool, &req->res.headers, H2O_STRLIT("access-control-allow-origin"),
0, NULL, H2O_STRLIT("*"));
@@ -318,7 +337,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
// Except for health check, wait for replicating state to be ready before allowing requests
// Follower or leader must have started AND data must also have been loaded
- if(path_without_query != "/health" && !self->http_server->get_replication_state()->is_ready()) {
+ if(path_without_query != "/health" && !h2o_handler->http_server->get_replication_state()->is_ready()) {
std::string message = "{ \"message\": \"Not Ready\"}";
return send_response(req, 503, message);
}
@@ -345,7 +364,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
}
route_path *rpath = nullptr;
- uint64_t route_hash = self->http_server->find_route(path_parts, http_method, &rpath);
+ uint64_t route_hash = h2o_handler->http_server->find_route(path_parts, http_method, &rpath);
if(route_hash == static_cast<uint64_t>(ROUTE_CODES::NOT_FOUND)) {
std::string message = "{ \"message\": \"Not Found\"}";
@@ -360,18 +379,27 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
}
}
- bool authenticated = self->http_server->auth_handler(query_map, *rpath, api_auth_key_sent);
+ bool authenticated = h2o_handler->http_server->auth_handler(query_map, *rpath, api_auth_key_sent);
if(!authenticated) {
std::string message = std::string("{\"message\": \"Forbidden - a valid `") + AUTH_HEADER +
"` header must be sent.\"}";
return send_response(req, 401, message);
}
- const std::string & req_body = std::string(req->entity.base, req->entity.len);
-
- http_req* request = new http_req(req, http_method, route_hash, query_map, req_body);
+ const std::string & body = std::string(req->entity.base, req->entity.len);
+ http_req* request = new http_req(req, rpath->http_method, route_hash, query_map, body);
http_res* response = new http_res();
+ // add custom generator with a dipose function for cleaning up resources
+ h2o_custom_generator_t* custom_generator = static_cast<h2o_custom_generator_t*>(
+ h2o_mem_alloc_shared(&req->pool, sizeof(*custom_generator), on_res_generator_dispose)
+ );
+ custom_generator->super = h2o_generator_t {response_proceed, nullptr};
+ custom_generator->req = request;
+ custom_generator->res = response;
+ custom_generator->req_handler = rpath->handler;
+ response->generator = &custom_generator->super;
+
// routes match and is an authenticated request
// do any additional pre-request middleware operations here
if(rpath->action == "keys:create") {
@@ -379,30 +407,100 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
request->metadata = StringUtils::randstring(AuthManager::KEY_LEN);
}
- // for writes, we defer to replication_state
- if(http_method == "POST" || http_method == "PUT" || http_method == "DELETE") {
- self->http_server->get_replication_state()->write(request, response);
+ if(req->proceed_req == nullptr) {
+ // Full request body is already available, so we don't care if handler is async or not
+ return process_request(request, response, rpath, h2o_handler);
+ } else {
+ // Only partial request body is available.
+ // If async_req is true, the request handler function will be invoked multiple times, for each chunk
+ request->streaming = rpath->async_req;
+ async_req_ctx_t* async_req_ctx = new async_req_ctx_t(request, response, h2o_handler, rpath);
+
+ //LOG(INFO) << "Partial request body length: " << req->entity.len;
+
+ req->write_req.cb = async_req_cb;
+ req->write_req.ctx = async_req_ctx;
+ int is_end_entity = 0;
+ req->proceed_req(req, req->entity.len, is_end_entity);
+ }
+
+ return 0;
+}
+
+int HttpServer::async_req_cb(void *ctx, h2o_iovec_t chunk, int is_end_stream) {
+ async_req_ctx_t* req_ctx = static_cast<async_req_ctx_t*>(ctx);
+
+ http_req* request = req_ctx->request;
+ http_res* response = req_ctx->response;
+
+ LOG(INFO) << "chunk.len: " << chunk.len << ", is_end_stream: " << is_end_stream;
+
+ request->chunk_length = chunk.len;
+
+ std::string chunk_str(chunk.base, chunk.len);
+ request->body += chunk_str;
+
+ if(request->streaming || is_end_stream) {
+ // Handler should be invoked for every chunk for streaming requests
+ // For a non streaming request, buffer body and invoke only at the end
+
+ // default value for response is NON_STREAMING
+ // for streaming requests, we have to set: START, IN_PROGRESS or END
+ if(request->streaming) {
+ if(is_end_stream) {
+ request->stream_state = "END";
+ } else if(request->stream_state == "NON_STREAMING") {
+ // `NON_STREAMING` default value indicates first chunk of streaming body
+ request->stream_state = "START";
+ } else {
+ request->stream_state = "IN_PROGRESS";
+ }
+ }
+
+ process_request(request, response, req_ctx->rpath, req_ctx->handler);
+ }
+
+ if(!is_end_stream && !request->streaming) {
+ // streaming requests will call proceed_req in an async fashion
+ // so we have to handle this only for non streaming
+ size_t written = chunk.len;
+ int stream_ended = 0;
+ request->_req->proceed_req(request->_req, written, stream_ended);
+ }
+
+ if (is_end_stream) {
+ // deletes only the container -- individual requests and response are deleted by response handler
+ delete req_ctx;
+ }
+
+ return 0;
+}
+
+int HttpServer::process_request(http_req* request, http_res* response, route_path *rpath,
+ const h2o_custom_req_handler_t *handler) {
+
+ // for writes, we delegate to replication_state to handle response
+ if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE") {
+ handler->http_server->get_replication_state()->write(request, response);
return 0;
}
+ // for reads, we will invoke the request handler and handle response as well
+
(rpath->handler)(*request, *response);
if(!rpath->async_res) {
// If a handler is marked as async res, it's responsible for sending the response itself in an async fashion
- // otherwise, we send the response on behalf of the handler
- self->http_server->send_response(request, response);
+ // otherwise, we send the whole body response on behalf of the handler
+ handler->http_server->send_response(request, response);
}
return 0;
}
-void HttpServer::send_message(const std::string & type, void* data) {
- message_dispatcher->send_message(type, data);
-}
-
-void HttpServer::send_response(http_req* request, const http_res* response) {
+void HttpServer::send_response(http_req* request, http_res* response) {
h2o_req_t* req = request->_req;
- h2o_generator_t generator = {NULL, NULL};
+ h2o_generator_t& generator = *response->generator;
h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
req->res.status = response->status_code;
@@ -411,9 +509,6 @@ void HttpServer::send_response(http_req* request, const http_res* response) {
nullptr, response->content_type_header.c_str(), response->content_type_header.size());
h2o_start_response(req, &generator);
h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
-
- delete request;
- delete response;
}
int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string & message) {
@@ -427,36 +522,40 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string
return 0;
}
+void HttpServer::send_message(const std::string & type, void* data) {
+ message_dispatcher->send_message(type, data);
+}
+
void HttpServer::set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const route_path& rpath,
const std::string& auth_key)) {
auth_handler = handler;
}
-void HttpServer::get(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {
+void HttpServer::get(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
- route_path rpath("GET", path_parts, handler, async);
+ route_path rpath("GET", path_parts, handler, async_req, async_res);
routes.emplace_back(rpath.route_hash(), rpath);
}
-void HttpServer::post(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {
+void HttpServer::post(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
- route_path rpath("POST", path_parts, handler, async);
+ route_path rpath("POST", path_parts, handler, async_req, async_res);
routes.emplace_back(rpath.route_hash(), rpath);
}
-void HttpServer::put(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {
+void HttpServer::put(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
- route_path rpath("PUT", path_parts, handler, async);
+ route_path rpath("PUT", path_parts, handler, async_req, async_res);
routes.emplace_back(rpath.route_hash(), rpath);
}
-void HttpServer::del(const std::string & path, bool (*handler)(http_req &, http_res &), bool async) {
+void HttpServer::del(const std::string & path, bool (*handler)(http_req &, http_res &), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
- route_path rpath("DELETE", path_parts, handler, async);
+ route_path rpath("DELETE", path_parts, handler, async_req, async_res);
routes.emplace_back(rpath.route_hash(), rpath);
}
diff --git a/src/index.cpp b/src/index.cpp
index e920c35e..95389432 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -784,8 +784,10 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
// every element in `query_suggestion` contains a token and its associated hits
std::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);
- /*for(size_t i=0; i < query_suggestion.size(); i++) {
- LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key;
+ /*LOG(INFO) << "n: " << n;
+ for(size_t i=0; i < query_suggestion.size(); i++) {
+ LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key << ", ids: "
+ << query_suggestion[i]->values->ids.getLength();
}*/
// initialize results with the starting element (for further intersection)
@@ -857,32 +859,13 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, si
searched_queries.push_back(query_suggestion);
+ //LOG(INFO) << "all_result_ids_len: " << all_result_ids_len << ", typo_tokens_threshold: " << typo_tokens_threshold;
if(all_result_ids_len >= typo_tokens_threshold) {
break;
}
}
}
-size_t Index::union_of_ids(std::vector<std::pair<uint32_t*, size_t>> & result_array_pairs,
- uint32_t **results_out) {
- uint32_t *results = nullptr;
- size_t results_length = 0;
-
- uint32_t *prev_results = nullptr;
- size_t prev_results_length = 0;
-
- for(const std::pair<uint32_t*, size_t> & result_array_pair: result_array_pairs) {
- results_length = ArrayUtils::or_scalar(prev_results, prev_results_length, result_array_pair.first,
- result_array_pair.second, &results);
- delete [] prev_results;
- prev_results = results;
- prev_results_length = results_length;
- }
-
- *results_out = results;
- return results_length;
-}
-
Option<uint32_t> Index::do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters) {
uint32_t* filter_ids = nullptr;
uint32_t filter_ids_length = 0;
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 788fde93..9e91b7d0 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -23,8 +23,8 @@ void master_server_routes() {
server->post("/collections/:collection/documents", post_add_document);
server->get("/collections/:collection/documents/search", get_search);
- server->post("/collections/:collection/documents/import", post_import_documents);
- server->get("/collections/:collection/documents/export", get_export_documents, true);
+ server->post("/collections/:collection/documents/import", post_import_documents, true, true);
+ server->get("/collections/:collection/documents/export", get_export_documents, false, true);
server->get("/collections/:collection/documents/:id", get_fetch_document);
server->del("/collections/:collection/documents/:id", del_remove_document);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index b0986752..03f54dc9 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -161,16 +161,12 @@ void ReplicationState::write(http_req* request, http_res* response) {
// To avoid ABA problem
task.expected_term = leader_term.load(butil::memory_order_relaxed);
+ //LOG(INFO) << ":::" << "body size before apply: " << request->body.size();
+
// Now the task is applied to the group, waiting for the result.
return node->apply(task);
}
-void ReplicationState::read(http_res* response) {
- // NOT USED:
- // For consistency, reads to followers could be rejected.
- // Currently, we don't do implement reads via raft.
-}
-
void ReplicationState::on_apply(braft::Iterator& iter) {
// A batch of tasks are committed, which must be processed through
// |iter|
@@ -186,6 +182,8 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
ReplicationClosure* c = dynamic_cast<ReplicationClosure*>(iter.done());
response = c->get_response();
request = c->get_request();
+
+ //LOG(INFO) << ":::" << "body size inside apply: " << request->body.size();
} else {
// Parse request from the log
response = new http_res;
@@ -217,6 +215,12 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
}
}
+void ReplicationState::read(http_res* response) {
+ // NOT USED:
+ // For consistency, reads to followers could be rejected.
+ // Currently, we don't do implement reads via raft.
+}
+
void* ReplicationState::save_snapshot(void* arg) {
SnapshotArg* sa = static_cast<SnapshotArg*>(arg);
std::unique_ptr<SnapshotArg> arg_guard(sa);
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 99af2e01..b7725320 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -68,49 +68,33 @@ Option<std::string> fetch_file_contents(const std::string & file_path) {
return Option<std::string>(content);
}
-void response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
- h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);
- custom_generator->handler(custom_generator->req, custom_generator->res, custom_generator->data);
+void stream_response(bool (*req_handler)(http_req* req, http_res* res, void* data),
+ http_req & request, http_res & response, void* data) {
+ h2o_req_t* req = request._req;
+ h2o_custom_generator_t *custom_generator = reinterpret_cast<h2o_custom_generator_t *>(response.generator);
+
+ if (request.stream_state == "START" || request.stream_state == "NON_STREAMING") {
+ req->res.status = response.status_code;
+ req->res.reason = http_res::get_status_reason(response.status_code);
+ h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL,
+ response.content_type_header.c_str(),
+ response.content_type_header.size());
+ h2o_start_response(req, &custom_generator->super);
+ }
- h2o_iovec_t body = h2o_strdup(&req->pool, custom_generator->res->body.c_str(), SIZE_MAX);
- const h2o_send_state_t state = custom_generator->res->final ?
- H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
+ custom_generator->res->final = (request.stream_state == "END" || request.stream_state == "NON_STREAMING");
+
+ h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);
+ const h2o_send_state_t state = custom_generator->res->final ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
h2o_send(req, &body, 1, state);
- if(custom_generator->res->final) {
- h2o_dispose_request(req);
+ if(state == H2O_SEND_STATE_FINAL) {
delete custom_generator->req;
delete custom_generator->res;
delete custom_generator;
}
}
-void response_stop(h2o_generator_t *generator, h2o_req_t *req) {
- h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);
-
- h2o_dispose_request(req);
- delete custom_generator->req;
- delete custom_generator->res;
- delete custom_generator;
-}
-
-void stream_response(bool (*handler)(http_req* req, http_res* res, void* data),
- http_req & request, http_res & response, void* data) {
- h2o_req_t* req = request._req;
- h2o_custom_generator_t* custom_generator = new h2o_custom_generator_t {
- h2o_generator_t {response_proceed, response_stop}, handler, &request, &response, data
- };
-
- req->res.status = response.status_code;
- req->res.reason = http_res::get_status_reason(response.status_code);
- h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL, response.content_type_header.c_str(),
- response.content_type_header.size());
- h2o_start_response(req, &custom_generator->super);
-
- h2o_iovec_t body = h2o_strdup(&req->pool, "", SIZE_MAX);
- h2o_send(req, &body, 1, H2O_SEND_STATE_IN_PROGRESS);
-}
-
void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.set_program_name("./typesense-server");
diff --git a/test/auth_manager_test.cpp b/test/auth_manager_test.cpp
index 7582cf11..76316289 100644
--- a/test/auth_manager_test.cpp
+++ b/test/auth_manager_test.cpp
@@ -163,12 +163,12 @@ TEST_F(AuthManagerTest, VerifyAuthentication) {
}
TEST_F(AuthManagerTest, GenerationOfAPIAction) {
- route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false);
- route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false);
- route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false);
- route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false);
- route_path rpath_doc_delete = route_path("DELETE", {"collections", ":collection", "documents", ":id"}, nullptr, false);
- route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false);
+ route_path rpath_search = route_path("GET", {"collections", ":collection", "documents", "search"}, nullptr, false, false);
+ route_path rpath_coll_get = route_path("GET", {"collections", ":collection"}, nullptr, false, false);
+ route_path rpath_coll_list = route_path("GET", {"collections"}, nullptr, false, false);
+ route_path rpath_keys_post = route_path("POST", {"keys"}, nullptr, false, false);
+ route_path rpath_doc_delete = route_path("DELETE", {"collections", ":collection", "documents", ":id"}, nullptr, false, false);
+ route_path rpath_override_upsert = route_path("PUT", {"collections", ":collection", "overrides", ":id"}, nullptr, false, false);
ASSERT_STREQ("documents:search", rpath_search._get_action().c_str());
ASSERT_STREQ("collections:get", rpath_coll_get._get_action().c_str());
diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp
index 1cc44ff1..9ca0ea5a 100644
--- a/test/string_utils_test.cpp
+++ b/test/string_utils_test.cpp
@@ -77,4 +77,9 @@ TEST(StringUtilsTest, ShouldSplitString) {
ASSERT_STREQ("{\"s\":\"Line one.\\nLine two.\\n\"}", lines[0].c_str());
ASSERT_STREQ("{\"s\":\"Line 1.\\nLine 2.\\n\"}", lines[1].c_str());
+
+ // empty string should produce empty list
+ std::vector<std::string> lines_empty;
+ StringUtils::split("", lines_empty, "\n");
+ ASSERT_TRUE(lines_empty.empty());
}