Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Remove use of doc meta key as it's not needed anymore.

kishorenccfc922c6f0ac
diff --git a/include/http_data.h b/include/http_data.h
index 2b0235f7..ca48e070 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -309,11 +309,6 @@ struct http_req {
}
};
-struct request_response {
- const std::shared_ptr<http_req>& req;
- const std::shared_ptr<http_res>& res;
-};
-
struct route_path {
std::string http_method;
std::vector<std::string> path_parts;
@@ -459,11 +454,11 @@ struct http_message_dispatcher {
}
};
-struct AsyncIndexArg {
+struct request_response_t {
std::shared_ptr<http_req> req;
std::shared_ptr<http_res> res;
- AsyncIndexArg(const std::shared_ptr<http_req> &req, const std::shared_ptr<http_res> &res): req(req), res(res) {
+ request_response_t(const std::shared_ptr<http_req> &req, const std::shared_ptr<http_res> &res): req(req), res(res) {
}
};
diff --git a/src/collection.cpp b/src/collection.cpp
index bf7bdd06..82570652 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -91,17 +91,6 @@ Option<doc_seq_id_t> Collection::to_doc(const std::string & json_str, nlohmann::
return Option<doc_seq_id_t>(400, "The `id` should not be empty.");
}
- bool meta_key_found = document.count(DOC_META_KEY) != 0;
- if(operation == CREATE && meta_key_found) {
- return Option<doc_seq_id_t>(400, "Document cannot contain a `" + std::string(DOC_META_KEY) + "` key.");
- }
-
- int dirty_values_integer = magic_enum::enum_integer(dirty_values);
- if(!meta_key_found) {
- document[DOC_META_KEY] = nlohmann::json::object();
- }
- document[DOC_META_KEY][DOC_META_DIRTY_VALUES_KEY] = dirty_values_integer;
-
if(document.count("id") == 0) {
if(operation == UPDATE) {
return Option<doc_seq_id_t>(400, "For update, the `id` key must be provided.");
@@ -180,10 +169,6 @@ Option<nlohmann::json> Collection::add(const std::string & json_str,
std::vector<std::string> json_lines = {json_str};
const nlohmann::json& res = add_many(json_lines, document, operation, id, dirty_values);
- if(document.count(Collection::DOC_META_KEY) != 0) {
- document.erase(Collection::DOC_META_KEY);
- }
-
if(!res["success"].get<bool>()) {
nlohmann::json res_doc;
@@ -410,8 +395,7 @@ void Collection::prune_document(nlohmann::json &document, const spp::sparse_hash
auto it = document.begin();
for(; it != document.end(); ) {
if (exclude_fields.count(it.key()) != 0 ||
- (!include_fields.empty() && include_fields.count(it.key()) == 0) ||
- document.count(Collection::DOC_META_KEY) != 0) {
+ (!include_fields.empty() && include_fields.count(it.key()) == 0)) {
it = document.erase(it);
} else {
++it;
@@ -1560,10 +1544,6 @@ Option<nlohmann::json> Collection::get(const std::string & id) const {
return Option<nlohmann::json>(500, "Error while parsing stored document.");
}
- if(document.count(Collection::DOC_META_KEY) != 0) {
- document.erase(Collection::DOC_META_KEY);
- }
-
return Option<nlohmann::json>(document);
}
@@ -2258,7 +2238,7 @@ Option<bool> Collection::check_and_update_schema(nlohmann::json& document, const
auto kv = document.begin();
for(; kv != document.end(); ) {
// we will not index the special "id" or doc meta keys
- if (search_schema.count(kv.key()) == 0 && kv.key() != "id" && kv.key() != Collection::DOC_META_KEY) {
+ if (search_schema.count(kv.key()) == 0 && kv.key() != "id") {
std::string field_type;
bool parseable = field::get_type(kv.value(), field_type);
if(!parseable) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 9be6c2e7..e2612819 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -202,17 +202,7 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {
return Option<bool>(false, "Bad JSON.");
}
- auto dirty_values = DIRTY_VALUES::REJECT;
-
- if (document.count(Collection::DOC_META_KEY) != 0 &&
- document[Collection::DOC_META_KEY].count(Collection::DOC_META_DIRTY_VALUES_KEY) != 0) {
-
- auto dirty_values_op = magic_enum::enum_cast<DIRTY_VALUES>(
- document[Collection::DOC_META_KEY][Collection::DOC_META_DIRTY_VALUES_KEY]);
- if(dirty_values_op.has_value()) {
- dirty_values = dirty_values_op.value();
- }
- }
+ auto dirty_values = DIRTY_VALUES::DROP;
num_valid_docs++;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 0760e5c5..c259b548 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -371,7 +371,6 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
}
if(it->Valid() && it->key().ToString().compare(0, seq_id_prefix.size(), seq_id_prefix) == 0) {
- // FIXME: remove meta key from doc
res->body = it->value().ToString();
it->Next();
@@ -1081,8 +1080,8 @@ bool del_key(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_re
bool raft_write_send_response(void *data) {
//LOG(INFO) << "raft_write_send_response called";
- AsyncIndexArg* index_arg = static_cast<AsyncIndexArg*>(data);
- std::unique_ptr<AsyncIndexArg> index_arg_guard(index_arg);
+ request_response_t* index_arg = static_cast<request_response_t*>(data);
+ std::unique_ptr<request_response_t> index_arg_guard(index_arg);
bool async_res = false;
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 73739c21..db99ea68 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -664,16 +664,14 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
//LOG(INFO) << "proxied_stream: " << custom_generator->response->proxied_stream;
//LOG(INFO) << "response.final: " << custom_generator->response->final;
+ custom_generator->res()->notify();
+
if(custom_generator->res()->proxied_stream) {
// request progression should not be tied to response generation
//LOG(INFO) << "Ignoring request proceed";
- custom_generator->res()->notify();
return ;
}
- // FIXME: should this be at the end of the function?
- custom_generator->res()->notify();
-
// if the request itself is async, we will proceed the request to fetch input content
if (custom_generator->rpath->async_req) {
auto stream_state = (custom_generator->res()->final) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 47d7ce67..4f57430d 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -159,7 +159,7 @@ void ReplicationState::write_to_leader(const std::shared_ptr<http_req>& request,
}
response->set_500("Could not find a leader.");
- auto replication_arg = new AsyncIndexArg{request, response};
+ auto replication_arg = new request_response_t{request, response};
replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);
return message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
}
@@ -229,7 +229,7 @@ void ReplicationState::write_to_leader(const std::shared_ptr<http_req>& request,
response->set_500(err);
}
- auto replication_arg = new AsyncIndexArg{request, response};
+ auto replication_arg = new request_response_t{request, response};
replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);
message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
});
@@ -285,8 +285,7 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
// Call http server thread for write and response back to client (if `response` is NOT null)
// We use a future to block current thread until the async flow finishes
- // FIXME: AsyncIndexArg lifecycle should be managed by caller
- AsyncIndexArg* replication_arg = new AsyncIndexArg{request_generated, response_generated};
+ request_response_t* replication_arg = new request_response_t{request_generated, response_generated};
message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
//LOG(INFO) << "Raft write waiting to proceed response->final=" << response_generated->final;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index ee405d83..68fedc6f 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -132,7 +132,7 @@ int init_logger(Config & config, const std::string & server_version) {
}
bool on_send_response(void *data) {
- request_response* req_res = static_cast<request_response*>(data);
+ request_response_t* req_res = static_cast<request_response_t*>(data);
server->send_response(req_res->req, req_res->res);
delete req_res;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d2f93050..64b5f7ea 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -91,27 +91,6 @@ TEST_F(CollectionTest, VerifyCountOfDocuments) {
ASSERT_EQ(DIRTY_VALUES::REJECT, collection->parse_dirty_values_option(empty_dirty_values));
}
-TEST_F(CollectionTest, MetaKeyChecks) {
- nlohmann::json results = collection->search("the", query_fields, "", {}, sort_fields, 0, 10).get();
- ASSERT_EQ(7, results["hits"].size());
- ASSERT_EQ(7, results["found"].get<int>());
-
- for(size_t i = 0; i < results["hits"].size(); i++) {
- nlohmann::json doc = results["hits"].at(i)["document"];
- ASSERT_EQ(0, doc.count(Collection::DOC_META_KEY));
- }
-
- // don't allow a document with meta key to be indexed since it is reserved
- nlohmann::json doc;
- doc["title"] = "foo bar";
- doc["points"] = 100;
- doc[Collection::DOC_META_KEY] = "override";
-
- auto op = collection->add(doc.dump());
- ASSERT_FALSE(op.ok());
- ASSERT_EQ("Document cannot contain a `$TSM$_` key.", op.error());
-}
-
TEST_F(CollectionTest, RetrieveADocumentById) {
Option<nlohmann::json> doc_option = collection->get("1");
ASSERT_TRUE(doc_option.ok());
@@ -124,9 +103,6 @@ TEST_F(CollectionTest, RetrieveADocumentById) {
id = doc["id"];
ASSERT_STREQ("foo", id.c_str());
- // returned document should not have internal doc meta key
- ASSERT_EQ(0, doc.count(Collection::DOC_META_KEY));
-
doc_option = collection->get("baz");
ASSERT_FALSE(doc_option.ok());
}
@@ -665,9 +641,7 @@ TEST_F(CollectionTest, MultiOccurrenceString) {
document["title"] = "The brown fox was the tallest of the lot and the quickest of the trot.";
document["points"] = 100;
- auto doc = coll_multi_string->add(document.dump()).get();
-
- ASSERT_EQ(0, doc.count(Collection::DOC_META_KEY));
+ coll_multi_string->add(document.dump()).get();
query_fields = {"title"};
nlohmann::json results = coll_multi_string->search("the", query_fields, "", {}, sort_fields, 0, 10, 1,