Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Delete documents matching a filter query.
kishorenceaea93c572e6
diff --git a/include/collection.h b/include/collection.hindex 0a3f17ce..bcfa8150 100644--- a/include/collection.h+++ b/include/collection.h@@ -184,6 +184,8 @@ private:void free_leaf_indices(std::vector<uint32_t*>& leaf_to_indices) const;+ Option<bool> parse_filter_query(const std::string& simple_filter_query, std::vector<filter>& filters);+public:Collection() = delete;@@ -259,12 +261,17 @@ public:const std::string& highlight_start_tag="<mark>",const std::string& highlight_end_tag="</mark>");+ Option<bool> get_filter_ids(const std::string & simple_filter_query,+ std::vector<std::pair<size_t, uint32_t*>>& index_ids);+Option<nlohmann::json> get(const std::string & id);bool doc_exists(const std::string & id);Option<std::string> remove(const std::string & id, bool remove_from_store = true);+ Option<bool> remove_if_found(uint32_t seq_id, bool remove_from_store = true);+Option<uint32_t> add_override(const override_t & override);Option<uint32_t> remove_override(const std::string & id);diff --git a/include/core_api.h b/include/core_api.hindex 713e5dc1..41c7ceeb 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -31,6 +31,8 @@ bool get_fetch_document(http_req& req, http_res& res);bool del_remove_document(http_req& req, http_res& res);+bool del_remove_documents(http_req& req, http_res& res);+// Aliasbool get_alias(http_req& req, http_res& res);diff --git a/include/core_api_utils.h b/include/core_api_utils.hnew file mode 100644index 00000000..ecbeee1a--- /dev/null+++ b/include/core_api_utils.h@@ -0,0 +1,14 @@+#pragma once++#include <cstdlib>+#include <vector>+#include "collection.h"++struct deletion_state_t {+ Collection* collection;+ std::vector<std::pair<size_t, uint32_t*>> index_ids;+ std::vector<size_t> offsets;+ size_t num_removed;+};++Option<bool> stateful_remove_docs(deletion_state_t* deletion_state, size_t batch_size, bool& done);\ No newline at end of filediff --git a/include/index.h b/include/index.hindex d3bff529..c74fb4dd 100644--- a/include/index.h+++ b/include/index.h@@ -146,8 +146,6 @@ private:void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;- Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);-void do_facets(std::vector<facet> & facets, facet_query_t & facet_query,const uint32_t* result_ids, size_t results_size);@@ -241,6 +239,8 @@ public:std::vector<std::vector<KV*>> & override_result_kvs,const size_t typo_tokens_threshold);+ Option<uint32_t> do_filtering(uint32_t** filter_ids_out, const std::vector<filter> & filters);+Option<uint32_t> remove(const uint32_t seq_id, const nlohmann::json & document);art_leaf* get_token_leaf(const std::string & field_name, const unsigned char* token, uint32_t token_len);diff --git a/src/collection.cpp b/src/collection.cppindex b58d94cd..55aff582 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -600,155 +600,10 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}}- // validate filter fields- std::vector<std::string> filter_blocks;- StringUtils::split(simple_filter_query, filter_blocks, "&&");-std::vector<filter> filters;- for(const std::string & filter_block: filter_blocks) {- // split into [field_name, value]- size_t found_index = filter_block.find(':');-- if(found_index == std::string::npos) {- return Option<nlohmann::json>(400, "Could not parse the filter query.");- }-- std::string&& field_name = filter_block.substr(0, found_index);- StringUtils::trim(field_name);-- if(search_schema.count(field_name) == 0) {- return Option<nlohmann::json>(404, "Could not find a filter field named `" + field_name + "` in the schema.");- }-- field _field = search_schema.at(field_name);- std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);- StringUtils::trim(raw_value);- filter f;-- // skip past optional `:=` operator, which has no meaning for non-string fields- if(!_field.is_string() && raw_value[0] == '=') {- size_t filter_value_index = 0;- while(raw_value[++filter_value_index] == ' ');- raw_value = raw_value.substr(filter_value_index);- }-- if(_field.is_integer() || _field.is_float()) {- // could be a single value or a list- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");-- for(const std::string & filter_value: filter_values) {- if(_field.is_int32()) {- if(!StringUtils::is_int32_t(filter_value)) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int32.");- }- }-- else if(_field.is_int64()) {- if(!StringUtils::is_int64_t(filter_value)) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int64.");- }- }-- else if(_field.is_float()) {- if(!StringUtils::is_float(filter_value)) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not a float.");- }- }- }-- f = {field_name, filter_values, EQUALS};- } else {- Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);- if(!op_comparator.ok()) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());- }-- // extract numerical value- std::string filter_value;- if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {- filter_value = raw_value.substr(1);- } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {- filter_value = raw_value.substr(2);- } else {- // EQUALS- filter_value = raw_value;- }-- filter_value = StringUtils::trim(filter_value);-- if(_field.is_int32()) {- if(!StringUtils::is_int32_t(filter_value)) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int32.");- }- }-- else if(_field.is_int64()) {- if(!StringUtils::is_int64_t(filter_value)) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not an int64.");- }- }-- else if(_field.is_float()) {- if(!StringUtils::is_float(filter_value)) {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Not a float.");- }- }-- f = {field_name, {filter_value}, op_comparator.get()};- }- } else if(_field.is_bool()) {- if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");-- for(std::string & filter_value: filter_values) {- if(filter_value != "true" && filter_value != "false") {- return Option<nlohmann::json>(400, "Values of filter field `" + _field.name +- "`: must be `true` or `false`.");- }-- filter_value = (filter_value == "true") ? "1" : "0";- }-- f = {field_name, filter_values, EQUALS};- } else {- if(raw_value != "true" && raw_value != "false") {- return Option<nlohmann::json>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");- }- std::string bool_value = (raw_value == "true") ? "1" : "0";- f = {field_name, {bool_value}, EQUALS};- }-- } else if(_field.is_string()) {- size_t filter_value_index = 0;- NUM_COMPARATOR str_comparator = CONTAINS;-- if(raw_value[0] == '=') {- if(!_field.facet) {- // EQUALS filtering on string is possible only on facet fields- return Option<nlohmann::json>(400, "To perform exact filtering, filter field `" +- _field.name + "` must be a facet field.");- }-- // string filter should be evaluated in strict "equals" mode- str_comparator = EQUALS;- while(raw_value[++filter_value_index] == ' ');- }-- if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {- std::vector<std::string> filter_values;- StringUtils::split(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values, ",");- f = {field_name, filter_values, str_comparator};- } else {- f = {field_name, {raw_value.substr(filter_value_index)}, str_comparator};- }- } else {- return Option<nlohmann::json>(400, "Error with filter field `" + _field.name + "`: Unidentified field type.");- }-- filters.push_back(f);+ Option<bool> parse_filter_op = parse_filter_query(simple_filter_query, filters);+ if(!parse_filter_op.ok()) {+ return Option<nlohmann::json>(parse_filter_op.code(), parse_filter_op.error());}// validate facet fields@@ -1297,6 +1152,29 @@ void Collection::aggregate_topster(size_t query_index, Topster &topster, Topster}}+Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,+ std::vector<std::pair<size_t, uint32_t*>>& index_ids) {+ std::vector<filter> filters;+ Option<bool> filter_op = parse_filter_query(simple_filter_query, filters);++ if(!filter_op.ok()) {+ return filter_op;+ }++ for(auto& index: indices) {+ uint32_t* filter_ids = nullptr;+ Option<uint32_t> op_filter_ids_length = index->do_filtering(&filter_ids, filters);+ if(!op_filter_ids_length.ok()) {+ return Option<bool>(op_filter_ids_length.code(), op_filter_ids_length.error());+ }++ size_t filter_ids_len = op_filter_ids_length.get();+ index_ids.emplace_back(filter_ids_len, filter_ids);+ }++ return Option<bool>(true);+}+bool Collection::facet_value_to_string(const facet &a_facet, const facet_count_t &facet_count,const nlohmann::json &document, std::string &value) {@@ -1611,6 +1489,30 @@ Option<std::string> Collection::remove(const std::string & id, const bool removereturn Option<std::string>(id);}+Option<bool> Collection::remove_if_found(uint32_t seq_id, const bool remove_from_store) {+ std::string parsed_document;+ StoreStatus doc_status = store->get(get_seq_id_key(seq_id), parsed_document);++ if(doc_status == StoreStatus::NOT_FOUND) {+ return Option<bool>(false);+ }++ if(doc_status == StoreStatus::ERROR) {+ return Option<bool>(500, "Error while fetching the document with seq id: " ++ std::to_string(seq_id));+ }++ nlohmann::json document;+ try {+ document = nlohmann::json::parse(parsed_document);+ } catch(...) {+ return Option<bool>(500, "Error while parsing stored document.");+ }++ remove_document(document, seq_id, remove_from_store);+ return Option<bool>(true);+}+Option<uint32_t> Collection::add_override(const override_t & override) {if(overrides.count("id") != 0) {return Option<uint32_t>(409, "There is already another entry with that `id`.");@@ -1756,3 +1658,157 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,const std::vector<Index *> &Collection::_get_indexes() const {return indices;}++Option<bool> Collection::parse_filter_query(const std::string& simple_filter_query,+ std::vector<filter>& filters) {+ std::vector<std::string> filter_blocks;+ StringUtils::split(simple_filter_query, filter_blocks, "&&");++ for(const std::string & filter_block: filter_blocks) {+ // split into [field_name, value]+ size_t found_index = filter_block.find(':');++ if(found_index == std::string::npos) {+ return Option<bool>(400, "Could not parse the filter query.");+ }++ std::string&& field_name = filter_block.substr(0, found_index);+ StringUtils::trim(field_name);++ if(search_schema.count(field_name) == 0) {+ return Option<bool>(404, "Could not find a filter field named `" + field_name + "` in the schema.");+ }++ field _field = search_schema.at(field_name);+ std::string&& raw_value = filter_block.substr(found_index+1, std::string::npos);+ StringUtils::trim(raw_value);+ filter f;++ // skip past optional `:=` operator, which has no meaning for non-string fields+ if(!_field.is_string() && raw_value[0] == '=') {+ size_t filter_value_index = 0;+ while(raw_value[++filter_value_index] == ' ');+ raw_value = raw_value.substr(filter_value_index);+ }++ if(_field.is_integer() || _field.is_float()) {+ // could be a single value or a list+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");++ for(const std::string & filter_value: filter_values) {+ if(_field.is_int32()) {+ if(!StringUtils::is_int32_t(filter_value)) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int32.");+ }+ }++ else if(_field.is_int64()) {+ if(!StringUtils::is_int64_t(filter_value)) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int64.");+ }+ }++ else if(_field.is_float()) {+ if(!StringUtils::is_float(filter_value)) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not a float.");+ }+ }+ }++ f = {field_name, filter_values, EQUALS};+ } else {+ Option<NUM_COMPARATOR> op_comparator = filter::extract_num_comparator(raw_value);+ if(!op_comparator.ok()) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: " + op_comparator.error());+ }++ // extract numerical value+ std::string filter_value;+ if(op_comparator.get() == LESS_THAN || op_comparator.get() == GREATER_THAN) {+ filter_value = raw_value.substr(1);+ } else if(op_comparator.get() == LESS_THAN_EQUALS || op_comparator.get() == GREATER_THAN_EQUALS) {+ filter_value = raw_value.substr(2);+ } else {+ // EQUALS+ filter_value = raw_value;+ }++ filter_value = StringUtils::trim(filter_value);++ if(_field.is_int32()) {+ if(!StringUtils::is_int32_t(filter_value)) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int32.");+ }+ }++ else if(_field.is_int64()) {+ if(!StringUtils::is_int64_t(filter_value)) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not an int64.");+ }+ }++ else if(_field.is_float()) {+ if(!StringUtils::is_float(filter_value)) {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Not a float.");+ }+ }++ f = {field_name, {filter_value}, op_comparator.get()};+ }+ } else if(_field.is_bool()) {+ if(raw_value[0] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(1, raw_value.size() - 2), filter_values, ",");++ for(std::string & filter_value: filter_values) {+ if(filter_value != "true" && filter_value != "false") {+ return Option<bool>(400, "Values of filter field `" + _field.name ++ "`: must be `true` or `false`.");+ }++ filter_value = (filter_value == "true") ? "1" : "0";+ }++ f = {field_name, filter_values, EQUALS};+ } else {+ if(raw_value != "true" && raw_value != "false") {+ return Option<bool>(400, "Value of filter field `" + _field.name + "` must be `true` or `false`.");+ }+ std::string bool_value = (raw_value == "true") ? "1" : "0";+ f = {field_name, {bool_value}, EQUALS};+ }++ } else if(_field.is_string()) {+ size_t filter_value_index = 0;+ NUM_COMPARATOR str_comparator = CONTAINS;++ if(raw_value[0] == '=') {+ if(!_field.facet) {+ // EQUALS filtering on string is possible only on facet fields+ return Option<bool>(400, "To perform exact filtering, filter field `" ++ _field.name + "` must be a facet field.");+ }++ // string filter should be evaluated in strict "equals" mode+ str_comparator = EQUALS;+ while(raw_value[++filter_value_index] == ' ');+ }++ if(raw_value[filter_value_index] == '[' && raw_value[raw_value.size() - 1] == ']') {+ std::vector<std::string> filter_values;+ StringUtils::split(raw_value.substr(filter_value_index+1, raw_value.size() - filter_value_index - 2), filter_values, ",");+ f = {field_name, filter_values, str_comparator};+ } else {+ f = {field_name, {raw_value.substr(filter_value_index)}, str_comparator};+ }+ } else {+ return Option<bool>(400, "Error with filter field `" + _field.name + "`: Unidentified field type.");+ }++ filters.push_back(f);+ }++ return Option<bool>(true);+}diff --git a/src/core_api.cpp b/src/core_api.cppindex 4551f24e..17f1fcc1 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -8,6 +8,7 @@#include "collection_manager.h"#include "system_metrics.h"#include "logger.h"+#include "core_api_utils.h"bool handle_authentication(std::map<std::string, std::string>& req_params, const route_path& rpath,const std::string& auth_key) {@@ -646,7 +647,10 @@ bool post_import_documents(http_req& req, http_res& res) {const size_t IMPORT_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);if(IMPORT_BATCH_SIZE == 0) {+ req.last_chunk_aggregate = true;+ res.final = true;res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");+ HttpServer::stream_response(req, res);return false;}@@ -854,6 +858,112 @@ bool del_remove_document(http_req & req, http_res & res) {return true;}+bool del_remove_documents(http_req & req, http_res & res) {+ // defaults: will get overridden later if needed+ res.content_type_header = "application/json";+ res.status_code = 200;++ // NOTE: this is a streaming response end-point so this handler will be called multiple times+ CollectionManager & collectionManager = CollectionManager::get_instance();+ Collection* collection = collectionManager.get_collection(req.params["collection"]);++ if(collection == nullptr) {+ req.last_chunk_aggregate = true;+ res.final = true;+ res.set_404();+ HttpServer::stream_response(req, res);+ return false;+ }++ const char *BATCH_SIZE = "batch_size";+ const char *FILTER_BY = "filter_by";++ if(req.params.count(BATCH_SIZE) == 0) {+ req.params[BATCH_SIZE] = "40";+ }++ if(req.params.count(FILTER_BY) == 0) {+ req.last_chunk_aggregate = true;+ res.final = true;+ res.set_400("Parameter `" + std::string(FILTER_BY) + "` must be provided.");+ HttpServer::stream_response(req, res);+ return false;+ }++ if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {+ req.last_chunk_aggregate = true;+ res.final = true;+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");+ HttpServer::stream_response(req, res);+ return false;+ }++ const size_t DELETE_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);++ if(DELETE_BATCH_SIZE == 0) {+ req.last_chunk_aggregate = true;+ res.final = true;+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");+ HttpServer::stream_response(req, res);+ return false;+ }++ std::string simple_filter_query;++ if(req.params.count(FILTER_BY) != 0) {+ simple_filter_query = req.params[FILTER_BY];+ }++ deletion_state_t* deletion_state = nullptr;++ if(req.data == nullptr) {+ deletion_state = new deletion_state_t{};+ auto filter_ids_op = collection->get_filter_ids(simple_filter_query, deletion_state->index_ids);++ if(!filter_ids_op.ok()) {+ res.set(filter_ids_op.code(), filter_ids_op.error());+ req.last_chunk_aggregate = true;+ res.final = true;+ HttpServer::stream_response(req, res);+ return false;+ }++ for(size_t i=0; i<deletion_state->index_ids.size(); i++) {+ deletion_state->offsets.push_back(0);+ }+ deletion_state->collection = collection;+ deletion_state->num_removed = 0;+ req.data = deletion_state;+ } else {+ deletion_state = static_cast<deletion_state_t*>(req.data);+ }++ bool done = true;+ Option<bool> remove_op = stateful_remove_docs(deletion_state, DELETE_BATCH_SIZE, done);++ if(!remove_op.ok()) {+ res.set(remove_op.code(), remove_op.error());+ req.last_chunk_aggregate = true;+ res.final = true;+ } else {+ if(!done) {+ req.last_chunk_aggregate = false;+ res.final = false;+ } else {+ nlohmann::json response;+ response["num_deleted"] = deletion_state->num_removed;++ req.last_chunk_aggregate = true;+ req.data = nullptr;+ res.body = response.dump();+ res.final = true;+ delete deletion_state;+ }+ }++ HttpServer::stream_response(req, res);+ return true;+}bool get_aliases(http_req & req, http_res & res) {CollectionManager & collectionManager = CollectionManager::get_instance();diff --git a/src/core_api_utils.cpp b/src/core_api_utils.cppnew file mode 100644index 00000000..b7bdcc2e--- /dev/null+++ b/src/core_api_utils.cpp@@ -0,0 +1,45 @@+#include "core_api_utils.h"++Option<bool> stateful_remove_docs(deletion_state_t* deletion_state, size_t batch_size, bool& done) {+ bool removed = true;+ size_t batch_count = 0;++ for(size_t i=0; i<deletion_state->index_ids.size(); i++) {+ std::pair<size_t, uint32_t*>& size_ids = deletion_state->index_ids[i];+ size_t ids_len = size_ids.first;+ uint32_t* ids = size_ids.second;++ size_t start_index = deletion_state->offsets[i];+ size_t batched_len = std::min(ids_len, (start_index+batch_size));++ for(size_t j=start_index; j<batched_len; j++) {+ Option<bool> remove_op = deletion_state->collection->remove_if_found(ids[j], true);++ if(!remove_op.ok()) {+ return remove_op;+ }++ removed = remove_op.get();+ if(removed) {+ deletion_state->num_removed++;+ }++ deletion_state->offsets[i]++;+ batch_count++;++ if(batch_count == batch_size) {+ goto END;+ }+ }+ }++ END:++ done = true;+ for(size_t i=0; i<deletion_state->index_ids.size(); i++) {+ size_t current_offset = deletion_state->offsets[i];+ done = done && (current_offset == deletion_state->index_ids[i].first);+ }++ return Option<bool>(removed);+}\ No newline at end of filediff --git a/src/http_server.cpp b/src/http_server.cppindex 653e7214..7f5c0d1c 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -494,7 +494,8 @@ int HttpServer::process_request(http_req* request, http_res* response, route_pat//LOG(INFO) << "process_request called";// for writes, we delegate to replication_state to handle response- if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE") {+ if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE" ||+ rpath->http_method == "PATCH") {handler->http_server->get_replication_state()->write(request, response);return 0;}@@ -618,7 +619,8 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {} else {// otherwise, call the handler since it will be the handler that will be producing content// (streaming response but not request)- custom_generator->rpath->handler(*custom_generator->request, *custom_generator->response);+ custom_generator->h2o_handler->http_server->defer_processing(*custom_generator->request,+ *custom_generator->response, 1);}}diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex ce6155b2..2dc441b0 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -121,7 +121,7 @@ void benchmark_reactjs_pages(char* file_path) {Collection *collection = collectionManager.get_collection("reactjs_pages");if(collection == nullptr) {- collection = collectionManager.create_collection("reactjs_pages", fields_to_index, "dummy_sorting_field").get();+ collection = collectionManager.create_collection("reactjs_pages", 4, fields_to_index, "dummy_sorting_field").get();}std::ifstream infile(file_path);diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 28e2cd79..1b4e037b 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -29,6 +29,7 @@ void master_server_routes() {server->get("/collections/:collection/documents/:id", get_fetch_document);server->del("/collections/:collection/documents/:id", del_remove_document);+ server->del("/collections/:collection/documents", del_remove_documents, false, true);server->get("/collections/:collection/overrides", get_overrides);server->get("/collections/:collection/overrides/:id", get_override);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex cbfc993a..9ad0cf8c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2457,3 +2457,49 @@ TEST_F(CollectionTest, ReturnsResultsBasedOnPerPageParam) {ASSERT_EQ(5, results["hits"].size());ASSERT_EQ(25, results["found"].get<int>());}++TEST_F(CollectionTest, RemoveIfFound) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }++ for(size_t i=0; i<10; i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = "Title " + std::to_string(i);+ doc["points"] = i;++ coll1->add(doc.dump());+ }++ auto res = coll1->search("*", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();++ ASSERT_EQ(10, res["found"].get<int>());++ // removing found doc+ Option<bool> found_op = coll1->remove_if_found(0);+ ASSERT_TRUE(found_op.ok());+ ASSERT_TRUE(found_op.get());++ auto get_op = coll1->get("0");+ ASSERT_FALSE(get_op.ok());+ ASSERT_EQ(404, get_op.code());++ // removing doc not found+ found_op = coll1->remove_if_found(100);+ ASSERT_TRUE(found_op.ok());+ ASSERT_FALSE(found_op.get());++ collectionManager.drop_collection("coll1");+}diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppnew file mode 100644index 00000000..3160636a--- /dev/null+++ b/test/core_api_utils_test.cpp@@ -0,0 +1,119 @@+#include <gtest/gtest.h>+#include "collection.h"+#include <vector>+#include <collection_manager.h>+#include "core_api_utils.h"++class CoreAPIUtilsTest : public ::testing::Test {+protected:+ Store *store;+ CollectionManager & collectionManager = CollectionManager::get_instance();++ std::vector<std::string> query_fields;+ std::vector<sort_by> sort_fields;++ void setupCollection() {+ std::string state_dir_path = "/tmp/typesense_test/core_api_utils";+ LOG(INFO) << "Truncating and creating: " << state_dir_path;+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key");+ collectionManager.load();+ }++ virtual void SetUp() {+ setupCollection();+ }++ virtual void TearDown() {+ collectionManager.dispose();+ delete store;+ }+};+++TEST_F(CoreAPIUtilsTest, StatefulRemoveDocs) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 2, fields, "points").get();+ }++ for(size_t i=0; i<100; i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = "Title " + std::to_string(i);+ doc["points"] = i;++ coll1->add(doc.dump());+ }++ bool done;+ deletion_state_t deletion_state;+ deletion_state.collection = coll1;+ deletion_state.num_removed = 0;++ // single document match++ coll1->get_filter_ids("points: 99", deletion_state.index_ids);+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {+ deletion_state.offsets.push_back(0);+ }++ stateful_remove_docs(&deletion_state, 5, done);+ ASSERT_EQ(1, deletion_state.num_removed);+ ASSERT_TRUE(done);++ // match 12 documents (multiple batches)+ deletion_state.index_ids.clear();+ deletion_state.offsets.clear();+ deletion_state.num_removed = 0;++ coll1->get_filter_ids("points:< 11", deletion_state.index_ids);+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {+ deletion_state.offsets.push_back(0);+ }++ stateful_remove_docs(&deletion_state, 4, done);+ ASSERT_EQ(4, deletion_state.num_removed);+ ASSERT_FALSE(done);++ stateful_remove_docs(&deletion_state, 4, done);+ ASSERT_EQ(8, deletion_state.num_removed);+ ASSERT_FALSE(done);++ stateful_remove_docs(&deletion_state, 4, done);+ ASSERT_EQ(11, deletion_state.num_removed);+ ASSERT_TRUE(done);++ // match 9 documents (multiple batches)+ deletion_state.index_ids.clear();+ deletion_state.offsets.clear();+ deletion_state.num_removed = 0;++ coll1->get_filter_ids("points:< 20", deletion_state.index_ids);+ for(size_t i=0; i<deletion_state.index_ids.size(); i++) {+ deletion_state.offsets.push_back(0);+ }++ stateful_remove_docs(&deletion_state, 7, done);+ ASSERT_EQ(7, deletion_state.num_removed);+ ASSERT_FALSE(done);++ stateful_remove_docs(&deletion_state, 7, done);+ ASSERT_EQ(9, deletion_state.num_removed);+ ASSERT_TRUE(done);++ // bad filter query+ auto op = coll1->get_filter_ids("bad filter", deletion_state.index_ids);+ ASSERT_FALSE(op.ok());+ ASSERT_STREQ("Could not parse the filter query.", op.error().c_str());++ collectionManager.drop_collection("coll1");+}\ No newline at end of file