Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Delete documents matching a filter query.

kishorenceaea93c572e6
diff --git a/include/collection.h b/include/collection.h
index 0a3f17ce..bcfa8150 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -184,6 +184,8 @@ private:
void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;
+ Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters);
+
public:
Collection() = delete;
@@ -259,12 +261,17 @@ public:
const std::string& highlight_start_tag="<mark>",
const std::string& highlight_end_tag="</mark>");
+ Option<bool> get_filter_ids(const std::string & simple_filter_query,
+ std::vector<std::pair<size_t, uint32_t*>>& index_ids);
+
Option<nlohmann::json> get(const std::string & id);
bool doc_exists(const std::string & id);
Option<std::string> remove(const std::string & id, bool remove_from_store = true);
+ Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);
+
Option<uint32_t> add_override(const override_t & override);
Option<uint32_t> remove_override(const std::string & id);
diff --git a/include/core_api.h b/include/core_api.h
index 713e5dc1..41c7ceeb 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -31,6 +31,8 @@ bool get_fetch_document(http_req& req, http_res& res);
bool del_remove_document(http_req& req, http_res& res);
+bool del_remove_documents(http_req& req, http_res& res);
+
// Alias
bool get_alias(http_req& req, http_res& res);
diff --git a/include/core_api_utils.h b/include/core_api_utils.h
new file mode 100644
index 00000000..ecbeee1a
--- /dev/null
+++ b/include/core_api_utils.h
@@ -0,0 +1,14 @@
+#pragma once
+
+#include <cstdlib>
+#include <vector>
+#include "collection.h"
+
+struct deletion_state_t {
+ Collection* collection;
+ std::vector<std::pair<size_t, uint32_t*>> index_ids;
+ std::vector<size_t> offsets;
+ size_t num_removed;
+};
+
+Option<bool> stateful_remove_docs(deletion_state_t* deletion_state, size_t batch_size, bool& done);
\ No newline at end of file
diff --git a/include/index.h b/include/index.h
index d3bff529..c74fb4dd 100644
--- a/include/index.h
+++ b/include/index.h
@@ -146,8 +146,6 @@ private:
void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;
- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
-
void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,
const uint32_t* result_ids, size_t results_size);
@@ -241,6 +239,8 @@ public:
std::vector<std::vector<KV*>> & override_result_kvs,
const size_t typo_tokens_threshold);
+ Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);
+
Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);
art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);
diff --git a/src/collection.cpp b/src/collection.cpp
index b58d94cd..55aff582 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -600,155 +600,10 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::
}
}
- // validate filter fields
- std::vector<std::string> filter_blocks;
- StringUtils::split(simple_filter_query, filter_blocks, "&&");
-
std::vector<filter> filters;
- for(const std::string & filter_block: filter_blocks) {
- // split into [field_name, value]
- size_t found_index = filter_block.find(':');
-
- if(found_index == std::string::npos) {
- return Option<nlohmann::json>(400, "Could not parse the filter query.");
- }
-
- std::string&& field_name = filter_block.substr(0, found_index);
- StringUtils::trim(field_name);
-
- if(search_schema.count(field_name) == 0) {
- return Option<nlohmann::json>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
- }
-
- field _field = search_schema.at(field_name);
- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
- StringUtils::trim(raw_value);
- filter f;
-
- // skip past optional `:=` operator, which has no meaning for non-string fields
- if(!_field.is_string() && raw_value[0] == '=') {
- size_t filter_value_index = 0;
- while(raw_value[++filter_value_index] == ' ');
- raw_value = raw_value.substr(filter_value_index);
- }
-
- if(_field.is_integer() || _field.is_float()) {
- // could be a single value or a list
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
-
- for(const std::string & filter_value: filter_values) {
- if(_field.is_int32()) {
- if(!StringUtils::is_int32_t(filter_value)) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int32.");
- }
- }
-
- else if(_field.is_int64()) {
- if(!StringUtils::is_int64_t(filter_value)) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int64.");
- }
- }
-
- else if(_field.is_float()) {
- if(!StringUtils::is_float(filter_value)) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not a float.");
- }
- }
- }
-
- f = {field_name, filter_values, EQUALS};
- } else {
- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
- if(!op_comparator.ok()) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
- }
-
- // extract numerical value
- std::string filter_value;
- if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {
- filter_value = raw_value.substr(1);
- } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {
- filter_value = raw_value.substr(2);
- } else {
- // EQUALS
- filter_value = raw_value;
- }
-
- filter_value = StringUtils::trim(filter_value);
-
- if(_field.is_int32()) {
- if(!StringUtils::is_int32_t(filter_value)) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int32.");
- }
- }
-
- else if(_field.is_int64()) {
- if(!StringUtils::is_int64_t(filter_value)) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int64.");
- }
- }
-
- else if(_field.is_float()) {
- if(!StringUtils::is_float(filter_value)) {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not a float.");
- }
- }
-
- f = {field_name, {filter_value}, op_comparator.get()};
- }
- } else if(_field.is_bool()) {
- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
-
- for(std::string & filter_value: filter_values) {
- if(filter_value != "true" && filter_value != "false") {
- return Option<nlohmann::json>(400, "Values of filter field `" + _field.name +
- "`: must be `true` or `false`.");
- }
-
- filter_value = (filter_value == "true") ? "1" : "0";
- }
-
- f = {field_name, filter_values, EQUALS};
- } else {
- if(raw_value != "true" && raw_value != "false") {
- return Option<nlohmann::json>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
- }
- std::string bool_value = (raw_value == "true") ? "1" : "0";
- f = {field_name, {bool_value}, EQUALS};
- }
-
- } else if(_field.is_string()) {
- size_t filter_value_index = 0;
- NUM_COMPARATOR str_comparator = CONTAINS;
-
- if(raw_value[0] == '=') {
- if(!_field.facet) {
- // EQUALS filtering on string is possible only on facet fields
- return Option<nlohmann::json>(400, "To perform exact filtering, filter field `" +
- _field.name + "` must be a facet field.");
- }
-
- // string filter should be evaluated in strict "equals" mode
- str_comparator = EQUALS;
- while(raw_value[++filter_value_index] == ' ');
- }
-
- if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
- std::vector<std::string> filter_values;
- StringUtils::split(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values, ",");
- f = {field_name, filter_values, str_comparator};
- } else {
- f = {field_name, {raw_value.substr(filter_value_index)}, str_comparator};
- }
- } else {
- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Unidentified field type.");
- }
-
- filters.push_back(f);
+ Option<bool> parse_filter_op = parse_filter_query(simple_filter_query, filters);
+ if(!parse_filter_op.ok()) {
+ return Option<nlohmann::json>(parse_filter_op.code(), parse_filter_op.error());
}
// validate facet fields
@@ -1297,6 +1152,29 @@ void Collection::aggregate_topster(size_t query_index, Topster &topster, Topster
}
}
+Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
+ std::vector<std::pair<size_t, uint32_t*>>& index_ids) {
+ std::vector<filter> filters;
+ Option<bool> filter_op = parse_filter_query(simple_filter_query, filters);
+
+ if(!filter_op.ok()) {
+ return filter_op;
+ }
+
+ for(auto& index: indices) {
+ uint32_t* filter_ids = nullptr;
+ Option<uint32_t> op_filter_ids_length = index->do_filtering(&filter_ids, filters);
+ if(!op_filter_ids_length.ok()) {
+ return Option<bool>(op_filter_ids_length.code(), op_filter_ids_length.error());
+ }
+
+ size_t filter_ids_len = op_filter_ids_length.get();
+ index_ids.emplace_back(filter_ids_len, filter_ids);
+ }
+
+ return Option<bool>(true);
+}
+
bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,
const nlohmann::json &document, std::string &value) {
@@ -1611,6 +1489,30 @@ Option<std::string> Collection::remove(const std::string & id, const bool remove
return Option<std::string>(id);
}
+Option<bool> Collection::remove_if_found(uint32_t seq_id, const bool remove_from_store) {
+ std::string parsed_document;
+ StoreStatus doc_status = store->get(get_seq_id_key(seq_id), parsed_document);
+
+ if(doc_status == StoreStatus::NOT_FOUND) {
+ return Option<bool>(false);
+ }
+
+ if(doc_status == StoreStatus::ERROR) {
+ return Option<bool>(500, "Error while fetching the document with seq id: " +
+ std::to_string(seq_id));
+ }
+
+ nlohmann::json document;
+ try {
+ document = nlohmann::json::parse(parsed_document);
+ } catch(...) {
+ return Option<bool>(500, "Error while parsing stored document.");
+ }
+
+ remove_document(document, seq_id, remove_from_store);
+ return Option<bool>(true);
+}
+
Option<uint32_t> Collection::add_override(const override_t & override) {
if(overrides.count("id") != 0) {
return Option<uint32_t>(409, "There is already another entry with that `id`.");
@@ -1756,3 +1658,157 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,
const std::vector<Index *> &Collection::_get_indexes() const {
return indices;
}
+
+Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,
+ std::vector<filter>& filters) {
+ std::vector<std::string> filter_blocks;
+ StringUtils::split(simple_filter_query, filter_blocks, "&&");
+
+ for(const std::string & filter_block: filter_blocks) {
+ // split into [field_name, value]
+ size_t found_index = filter_block.find(':');
+
+ if(found_index == std::string::npos) {
+ return Option<bool>(400, "Could not parse the filter query.");
+ }
+
+ std::string&& field_name = filter_block.substr(0, found_index);
+ StringUtils::trim(field_name);
+
+ if(search_schema.count(field_name) == 0) {
+ return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");
+ }
+
+ field _field = search_schema.at(field_name);
+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);
+ StringUtils::trim(raw_value);
+ filter f;
+
+ // skip past optional `:=` operator, which has no meaning for non-string fields
+ if(!_field.is_string() && raw_value[0] == '=') {
+ size_t filter_value_index = 0;
+ while(raw_value[++filter_value_index] == ' ');
+ raw_value = raw_value.substr(filter_value_index);
+ }
+
+ if(_field.is_integer() || _field.is_float()) {
+ // could be a single value or a list
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+
+ for(const std::string & filter_value: filter_values) {
+ if(_field.is_int32()) {
+ if(!StringUtils::is_int32_t(filter_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int32.");
+ }
+ }
+
+ else if(_field.is_int64()) {
+ if(!StringUtils::is_int64_t(filter_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int64.");
+ }
+ }
+
+ else if(_field.is_float()) {
+ if(!StringUtils::is_float(filter_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not a float.");
+ }
+ }
+ }
+
+ f = {field_name, filter_values, EQUALS};
+ } else {
+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);
+ if(!op_comparator.ok()) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());
+ }
+
+ // extract numerical value
+ std::string filter_value;
+ if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {
+ filter_value = raw_value.substr(1);
+ } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {
+ filter_value = raw_value.substr(2);
+ } else {
+ // EQUALS
+ filter_value = raw_value;
+ }
+
+ filter_value = StringUtils::trim(filter_value);
+
+ if(_field.is_int32()) {
+ if(!StringUtils::is_int32_t(filter_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int32.");
+ }
+ }
+
+ else if(_field.is_int64()) {
+ if(!StringUtils::is_int64_t(filter_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int64.");
+ }
+ }
+
+ else if(_field.is_float()) {
+ if(!StringUtils::is_float(filter_value)) {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not a float.");
+ }
+ }
+
+ f = {field_name, {filter_value}, op_comparator.get()};
+ }
+ } else if(_field.is_bool()) {
+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");
+
+ for(std::string & filter_value: filter_values) {
+ if(filter_value != "true" && filter_value != "false") {
+ return Option<bool>(400, "Values of filter field `" + _field.name +
+ "`: must be `true` or `false`.");
+ }
+
+ filter_value = (filter_value == "true") ? "1" : "0";
+ }
+
+ f = {field_name, filter_values, EQUALS};
+ } else {
+ if(raw_value != "true" && raw_value != "false") {
+ return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");
+ }
+ std::string bool_value = (raw_value == "true") ? "1" : "0";
+ f = {field_name, {bool_value}, EQUALS};
+ }
+
+ } else if(_field.is_string()) {
+ size_t filter_value_index = 0;
+ NUM_COMPARATOR str_comparator = CONTAINS;
+
+ if(raw_value[0] == '=') {
+ if(!_field.facet) {
+ // EQUALS filtering on string is possible only on facet fields
+ return Option<bool>(400, "To perform exact filtering, filter field `" +
+ _field.name + "` must be a facet field.");
+ }
+
+ // string filter should be evaluated in strict "equals" mode
+ str_comparator = EQUALS;
+ while(raw_value[++filter_value_index] == ' ');
+ }
+
+ if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {
+ std::vector<std::string> filter_values;
+ StringUtils::split(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values, ",");
+ f = {field_name, filter_values, str_comparator};
+ } else {
+ f = {field_name, {raw_value.substr(filter_value_index)}, str_comparator};
+ }
+ } else {
+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Unidentified field type.");
+ }
+
+ filters.push_back(f);
+ }
+
+ return Option<bool>(true);
+}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 4551f24e..17f1fcc1 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -8,6 +8,7 @@
#include "collection_manager.h"
#include "system_metrics.h"
#include "logger.h"
+#include "core_api_utils.h"
bool handle_authentication(std::map<std::string, std::string>& req_params, const route_path& rpath,
const std::string& auth_key) {
@@ -646,7 +647,10 @@ bool post_import_documents(http_req& req, http_res& res) {
const size_t IMPORT_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);
if(IMPORT_BATCH_SIZE == 0) {
+ req.last_chunk_aggregate = true;
+ res.final = true;
res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
+ HttpServer::stream_response(req, res);
return false;
}
@@ -854,6 +858,112 @@ bool del_remove_document(http_req & req, http_res & res) {
return true;
}
+bool del_remove_documents(http_req & req, http_res & res) {
+ // defaults: will get overridden later if needed
+ res.content_type_header = "application/json";
+ res.status_code = 200;
+
+ // NOTE: this is a streaming response end-point so this handler will be called multiple times
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ Collection* collection = collectionManager.get_collection(req.params["collection"]);
+
+ if(collection == nullptr) {
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ res.set_404();
+ HttpServer::stream_response(req, res);
+ return false;
+ }
+
+ const char *BATCH_SIZE = "batch_size";
+ const char *FILTER_BY = "filter_by";
+
+ if(req.params.count(BATCH_SIZE) == 0) {
+ req.params[BATCH_SIZE] = "40";
+ }
+
+ if(req.params.count(FILTER_BY) == 0) {
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ res.set_400("Parameter `" + std::string(FILTER_BY) + "` must be provided.");
+ HttpServer::stream_response(req, res);
+ return false;
+ }
+
+ if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
+ HttpServer::stream_response(req, res);
+ return false;
+ }
+
+ const size_t DELETE_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);
+
+ if(DELETE_BATCH_SIZE == 0) {
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
+ HttpServer::stream_response(req, res);
+ return false;
+ }
+
+ std::string simple_filter_query;
+
+ if(req.params.count(FILTER_BY) != 0) {
+ simple_filter_query = req.params[FILTER_BY];
+ }
+
+ deletion_state_t* deletion_state = nullptr;
+
+ if(req.data == nullptr) {
+ deletion_state = new deletion_state_t{};
+ auto filter_ids_op = collection->get_filter_ids(simple_filter_query, deletion_state->index_ids);
+
+ if(!filter_ids_op.ok()) {
+ res.set(filter_ids_op.code(), filter_ids_op.error());
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ HttpServer::stream_response(req, res);
+ return false;
+ }
+
+ for(size_t i=0; i<deletion_state->index_ids.size(); i++) {
+ deletion_state->offsets.push_back(0);
+ }
+ deletion_state->collection = collection;
+ deletion_state->num_removed = 0;
+ req.data = deletion_state;
+ } else {
+ deletion_state = static_cast<deletion_state_t*>(req.data);
+ }
+
+ bool done = true;
+ Option<bool> remove_op = stateful_remove_docs(deletion_state, DELETE_BATCH_SIZE, done);
+
+ if(!remove_op.ok()) {
+ res.set(remove_op.code(), remove_op.error());
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ } else {
+ if(!done) {
+ req.last_chunk_aggregate = false;
+ res.final = false;
+ } else {
+ nlohmann::json response;
+ response["num_deleted"] = deletion_state->num_removed;
+
+ req.last_chunk_aggregate = true;
+ req.data = nullptr;
+ res.body = response.dump();
+ res.final = true;
+ delete deletion_state;
+ }
+ }
+
+ HttpServer::stream_response(req, res);
+ return true;
+}
bool get_aliases(http_req & req, http_res & res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
diff --git a/src/core_api_utils.cpp b/src/core_api_utils.cpp
new file mode 100644
index 00000000..b7bdcc2e
--- /dev/null
+++ b/src/core_api_utils.cpp
@@ -0,0 +1,45 @@
+#include "core_api_utils.h"
+
+Option<bool> stateful_remove_docs(deletion_state_t* deletion_state, size_t batch_size, bool& done) {
+ bool removed = true;
+ size_t batch_count = 0;
+
+ for(size_t i=0; i<deletion_state->index_ids.size(); i++) {
+ std::pair<size_t, uint32_t*>& size_ids = deletion_state->index_ids[i];
+ size_t ids_len = size_ids.first;
+ uint32_t* ids = size_ids.second;
+
+ size_t start_index = deletion_state->offsets[i];
+ size_t batched_len = std::min(ids_len, (start_index+batch_size));
+
+ for(size_t j=start_index; j<batched_len; j++) {
+ Option<bool> remove_op = deletion_state->collection->remove_if_found(ids[j], true);
+
+ if(!remove_op.ok()) {
+ return remove_op;
+ }
+
+ removed = remove_op.get();
+ if(removed) {
+ deletion_state->num_removed++;
+ }
+
+ deletion_state->offsets[i]++;
+ batch_count++;
+
+ if(batch_count == batch_size) {
+ goto END;
+ }
+ }
+ }
+
+ END:
+
+ done = true;
+ for(size_t i=0; i<deletion_state->index_ids.size(); i++) {
+ size_t current_offset = deletion_state->offsets[i];
+ done = done && (current_offset == deletion_state->index_ids[i].first);
+ }
+
+ return Option<bool>(removed);
+}
\ No newline at end of file
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 653e7214..7f5c0d1c 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -494,7 +494,8 @@ int HttpServer::process_request(http_req* request, http_res* response, route_pat
//LOG(INFO) << "process_request called";
// for writes, we delegate to replication_state to handle response
- if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE") {
+ if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE" ||
+ rpath->http_method == "PATCH") {
handler->http_server->get_replication_state()->write(request, response);
return 0;
}
@@ -618,7 +619,8 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
} else {
// otherwise, call the handler since it will be the handler that will be producing content
// (streaming response but not request)
- custom_generator->rpath->handler(*custom_generator->request, *custom_generator->response);
+ custom_generator->h2o_handler->http_server->defer_processing(*custom_generator->request,
+ *custom_generator->response, 1);
}
}
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index ce6155b2..2dc441b0 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -121,7 +121,7 @@ void benchmark_reactjs_pages(char* file_path) {
Collection *collection = collectionManager.get_collection("reactjs_pages");
if(collection == nullptr) {
- collection = collectionManager.create_collection("reactjs_pages", fields_to_index, "dummy_sorting_field").get();
+ collection = collectionManager.create_collection("reactjs_pages", 4, fields_to_index, "dummy_sorting_field").get();
}
std::ifstream infile(file_path);
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 28e2cd79..1b4e037b 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -29,6 +29,7 @@ void master_server_routes() {
server->get("/collections/:collection/documents/:id", get_fetch_document);
server->del("/collections/:collection/documents/:id", del_remove_document);
+ server->del("/collections/:collection/documents", del_remove_documents, false, true);
server->get("/collections/:collection/overrides", get_overrides);
server->get("/collections/:collection/overrides/:id", get_override);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index cbfc993a..9ad0cf8c 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2457,3 +2457,49 @@ TEST_F(CollectionTest, ReturnsResultsBasedOnPerPageParam) {
ASSERT_EQ(5, results["hits"].size());
ASSERT_EQ(25, results["found"].get<int>());
}
+
+TEST_F(CollectionTest, RemoveIfFound) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, true),
+ field("points", field_types::INT32, false)};
+
+ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if (coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
+ }
+
+ for(size_t i=0; i<10; i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = "Title " + std::to_string(i);
+ doc["points"] = i;
+
+ coll1->add(doc.dump());
+ }
+
+ auto res = coll1->search("*", {"title"}, "", {}, sort_fields, 0, 10, 1,
+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();
+
+ ASSERT_EQ(10, res["found"].get<int>());
+
+ // removing found doc
+ Option<bool> found_op = coll1->remove_if_found(0);
+ ASSERT_TRUE(found_op.ok());
+ ASSERT_TRUE(found_op.get());
+
+ auto get_op = coll1->get("0");
+ ASSERT_FALSE(get_op.ok());
+ ASSERT_EQ(404, get_op.code());
+
+ // removing doc not found
+ found_op = coll1->remove_if_found(100);
+ ASSERT_TRUE(found_op.ok());
+ ASSERT_FALSE(found_op.get());
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
new file mode 100644
index 00000000..3160636a
--- /dev/null
+++ b/test/core_api_utils_test.cpp
@@ -0,0 +1,119 @@
+#include <gtest/gtest.h>
+#include "collection.h"
+#include <vector>
+#include <collection_manager.h>
+#include "core_api_utils.h"
+
+class CoreAPIUtilsTest : public ::testing::Test {
+protected:
+ Store *store;
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+
+ std::vector<std::string> query_fields;
+ std::vector<sort_by> sort_fields;
+
+ void setupCollection() {
+ std::string state_dir_path = "/tmp/typesense_test/core_api_utils";
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
+ store = new Store(state_dir_path);
+ collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.load();
+ }
+
+ virtual void SetUp() {
+ setupCollection();
+ }
+
+ virtual void TearDown() {
+ collectionManager.dispose();
+ delete store;
+ }
+};
+
+
+TEST_F(CoreAPIUtilsTest, StatefulRemoveDocs) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 2, fields, "points").get();
+ }
+
+ for(size_t i=0; i<100; i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = "Title " + std::to_string(i);
+ doc["points"] = i;
+
+ coll1->add(doc.dump());
+ }
+
+ bool done;
+ deletion_state_t deletion_state;
+ deletion_state.collection = coll1;
+ deletion_state.num_removed = 0;
+
+ // single document match
+
+ coll1->get_filter_ids("points: 99", deletion_state.index_ids);
+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {
+ deletion_state.offsets.push_back(0);
+ }
+
+ stateful_remove_docs(&deletion_state, 5, done);
+ ASSERT_EQ(1, deletion_state.num_removed);
+ ASSERT_TRUE(done);
+
+ // match 12 documents (multiple batches)
+ deletion_state.index_ids.clear();
+ deletion_state.offsets.clear();
+ deletion_state.num_removed = 0;
+
+ coll1->get_filter_ids("points:< 11", deletion_state.index_ids);
+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {
+ deletion_state.offsets.push_back(0);
+ }
+
+ stateful_remove_docs(&deletion_state, 4, done);
+ ASSERT_EQ(4, deletion_state.num_removed);
+ ASSERT_FALSE(done);
+
+ stateful_remove_docs(&deletion_state, 4, done);
+ ASSERT_EQ(8, deletion_state.num_removed);
+ ASSERT_FALSE(done);
+
+ stateful_remove_docs(&deletion_state, 4, done);
+ ASSERT_EQ(11, deletion_state.num_removed);
+ ASSERT_TRUE(done);
+
+ // match 9 documents (multiple batches)
+ deletion_state.index_ids.clear();
+ deletion_state.offsets.clear();
+ deletion_state.num_removed = 0;
+
+ coll1->get_filter_ids("points:< 20", deletion_state.index_ids);
+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {
+ deletion_state.offsets.push_back(0);
+ }
+
+ stateful_remove_docs(&deletion_state, 7, done);
+ ASSERT_EQ(7, deletion_state.num_removed);
+ ASSERT_FALSE(done);
+
+ stateful_remove_docs(&deletion_state, 7, done);
+ ASSERT_EQ(9, deletion_state.num_removed);
+ ASSERT_TRUE(done);
+
+ // bad filter query
+ auto op = coll1->get_filter_ids("bad filter", deletion_state.index_ids);
+ ASSERT_FALSE(op.ok());
+ ASSERT_STREQ("Could not parse the filter query.", op.error().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
\ No newline at end of file