Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Update documents using `filter_by`. (#845)

Harpreet Sangar5f034f8b3b3f
diff --git a/include/collection.h b/include/collection.h
index 6b3d9e71..7017098f 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -370,6 +370,11 @@ public:
const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT,
const bool& return_doc=false, const bool& return_id=false);
+ Option<nlohmann::json> update_matching_filter(const std::string& filter_query,
+ const std::string & json_str,
+ std::string& req_dirty_values,
+ const int batch_size = 1000);
+
Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,
const std::string & filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
@@ -414,8 +419,8 @@ public:
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0) const;
- Option<bool> get_filter_ids(const std::string & simple_filter_query,
- std::vector<std::pair<size_t, uint32_t*>>& index_ids);
+ Option<bool> get_filter_ids(const std::string & filter_query,
+ std::vector<std::pair<size_t, uint32_t*>>& index_ids) const;
Option<nlohmann::json> get(const std::string & id) const;
diff --git a/include/core_api.h b/include/core_api.h
index 8130434c..0bc1a615 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -33,6 +33,8 @@ bool post_add_document(const std::shared_ptr<http_req>& req, const std::shared_p
bool patch_update_document(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool patch_update_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
bool post_import_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
bool get_fetch_document(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/include/field.h b/include/field.h
index b8dbca82..242f8f9b 100644
--- a/include/field.h
+++ b/include/field.h
@@ -513,7 +513,7 @@ struct filter {
std::string& processed_filter_val,
NUM_COMPARATOR& num_comparator);
- static Option<bool> parse_filter_query(const std::string& simple_filter_query,
+ static Option<bool> parse_filter_query(const std::string& filter_query,
const tsl::htrie_map<char, field>& search_schema,
const Store* store,
const std::string& doc_id_prefix,
diff --git a/src/collection.cpp b/src/collection.cpp
index b6ef3405..ae30fbf3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -322,6 +322,100 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
return resp_summary;
}
+Option<nlohmann::json> Collection::update_matching_filter(const std::string& filter_query,
+ const std::string & json_str,
+ std::string& req_dirty_values,
+ const int batch_size) {
+ auto _filter_query = filter_query;
+ StringUtils::trim(_filter_query);
+
+ if (_filter_query.empty()) {
+ nlohmann::json resp_summary;
+ resp_summary["num_updated"] = 0;
+ return Option(resp_summary);
+ }
+
+ const auto& dirty_values = parse_dirty_values_option(req_dirty_values);
+ size_t docs_updated_count;
+ nlohmann::json update_document, dummy;
+
+ try {
+ update_document = nlohmann::json::parse(json_str);
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ return Option<nlohmann::json>(400, std::string("Bad JSON: ") + e.what());
+ }
+
+ std::vector<std::string> buffer;
+ buffer.reserve(batch_size);
+
+ if (_filter_query == "*") {
+ // Get an iterator from rocksdb and iterate over all the documents present in the collection.
+ std::string iter_upper_bound_key = get_seq_id_collection_prefix() + "`";
+ auto iter_upper_bound = new rocksdb::Slice(iter_upper_bound_key);
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ const std::string seq_id_prefix = get_seq_id_collection_prefix();
+ rocksdb::Iterator* it = collectionManager.get_store()->scan(seq_id_prefix, iter_upper_bound);
+
+ while(it->Valid()) {
+ // Generate a batch of documents to be ingested by add_many.
+ for (int buffer_counter = 0; buffer_counter < batch_size && it->Valid();) {
+ auto json_doc_str = it->value().ToString();
+ it->Next();
+ nlohmann::json existing_document;
+ try {
+ existing_document = nlohmann::json::parse(json_doc_str);
+ } catch(...) {
+ continue; // Don't add into buffer.
+ }
+
+ update_document["id"] = existing_document["id"].get<std::string>();
+ buffer.push_back(update_document.dump());
+ buffer_counter++;
+ }
+
+ auto res = add_many(buffer, dummy, index_operation_t::UPDATE, "", dirty_values);
+ docs_updated_count += res["num_imported"].get<size_t>();
+ buffer.clear();
+ }
+
+ delete iter_upper_bound;
+ delete it;
+ } else {
+ std::vector<std::pair<size_t, uint32_t*>> filter_ids;
+ auto filter_ids_op = get_filter_ids(_filter_query, filter_ids);
+ if(!filter_ids_op.ok()) {
+ return Option<nlohmann::json>(filter_ids_op.code(), filter_ids_op.error());
+ }
+
+ for (size_t i = 0; i < filter_ids[0].first;) {
+ for (int buffer_counter = 0; buffer_counter < batch_size && i < filter_ids[0].first;) {
+ uint32_t seq_id = *(filter_ids[0].second + i++);
+ nlohmann::json existing_document;
+
+ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), existing_document);
+ if (!get_doc_op.ok()) {
+ continue;
+ }
+
+ update_document["id"] = existing_document["id"].get<std::string>();
+ buffer.push_back(update_document.dump());
+ buffer_counter++;
+ }
+
+ auto res = add_many(buffer, dummy, index_operation_t::UPDATE, "", dirty_values);
+ docs_updated_count += res["num_imported"].get<size_t>();
+ buffer.clear();
+ }
+
+ delete [] filter_ids[0].second;
+ }
+
+ nlohmann::json resp_summary;
+ resp_summary["num_updated"] = docs_updated_count;
+ return Option(resp_summary);
+}
+
bool Collection::is_exceeding_memory_threshold() const {
return SystemMetrics::used_memory_ratio() > max_memory_ratio;
}
@@ -2225,13 +2319,13 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K
}
}
-Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,
- std::vector<std::pair<size_t, uint32_t*>>& index_ids) {
+Option<bool> Collection::get_filter_ids(const std::string & filter_query,
+ std::vector<std::pair<size_t, uint32_t*>>& index_ids) const {
std::shared_lock lock(mutex);
const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";
filter_node_t* filter_tree_root = nullptr;
- Option<bool> filter_op = filter::parse_filter_query(simple_filter_query, search_schema,
+ Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,
store, doc_id_prefix, filter_tree_root);
if(!filter_op.ok()) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 1599ddd2..09060a1e 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -944,6 +944,38 @@ bool patch_update_document(const std::shared_ptr<http_req>& req, const std::shar
return true;
}
+bool patch_update_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ const char *FILTER_BY = "filter_by";
+ std::string filter_query;
+ if(req->params.count(FILTER_BY) == 0) {
+ res->set_400("Parameter `" + std::string(FILTER_BY) + "` must be provided.");
+ return false;
+ } else {
+ filter_query = req->params[FILTER_BY];
+ }
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ auto collection = collectionManager.get_collection(req->params["collection"]);
+ if(collection == nullptr) {
+ res->set_404();
+ return false;
+ }
+
+ const char* DIRTY_VALUES_PARAM = "dirty_values";
+ if(req->params.count(DIRTY_VALUES_PARAM) == 0) {
+ req->params[DIRTY_VALUES_PARAM] = ""; // set it empty as default will depend on whether schema is enabled
+ }
+
+ auto update_op = collection->update_matching_filter(filter_query, req->body, req->params[DIRTY_VALUES_PARAM]);
+ if(update_op.ok()) {
+ res->set_200(update_op.get().dump());
+ } else {
+ res->set(update_op.code(), update_op.error());
+ }
+
+ return update_op.ok();
+}
+
bool get_fetch_document(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
std::string doc_id = req->params["id"];
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 79479ab7..b32057ed 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -30,6 +30,7 @@ void master_server_routes() {
server->get("/collections/:collection/documents/:id", get_fetch_document);
server->patch("/collections/:collection/documents/:id", patch_update_document);
+ server->patch("/collections/:collection/documents", patch_update_documents);
server->del("/collections/:collection/documents/:id", del_remove_document);
server->get("/collections/:collection/overrides", get_overrides);
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 9aa327d9..2e547041 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2528,6 +2528,124 @@ TEST_F(CollectionTest, UpdateDocument) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionTest, UpdateDocuments) {
+ nlohmann::json schema = R"({
+ "name": "update_docs_collection",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "user_name", "type": "string", "facet": true},
+ {"name": "likes", "type": "int32"},
+ {"name": "content", "type": "object"}
+ ],
+ "default_sorting_field": "likes"
+ })"_json;
+
+ Collection *update_docs_collection = collectionManager.get_collection("update_docs_collection").get();
+ if (update_docs_collection == nullptr) {
+ auto op = CollectionManager::create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ update_docs_collection = op.get();
+ }
+
+ std::vector<std::string> json_lines = {
+ R"({"user_name": "fat_cat","likes": 5215,"content": {"title": "cat data 1", "body": "cd1"}})",
+ R"({"user_name": "fast_dog","likes": 273,"content": {"title": "dog data 1", "body": "dd1"}})",
+ R"({"user_name": "fat_cat","likes": 2133,"content": {"title": "cat data 2", "body": "cd2"}})",
+ R"({"user_name": "fast_dog","likes": 9754,"content": {"title": "dog data 2", "body": "dd2"}})",
+ R"({"user_name": "fast_dog","likes": 576,"content": {"title": "dog data 3", "body": "dd3"}})"
+ };
+
+ for (auto const& json: json_lines){
+ auto add_op = update_docs_collection->add(json);
+ if (!add_op.ok()) {
+ std::cout << add_op.error() << std::endl;
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ std::vector<sort_by> sort_fields = { sort_by("likes", "DESC") };
+
+ auto res = update_docs_collection->search("cat data", {"content"}, "", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(2, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ("fat_cat", res["hits"][i]["document"]["user_name"].get<std::string>());
+ }
+
+ nlohmann::json document;
+ document["user_name"] = "slim_cat";
+ std::string dirty_values;
+
+ auto update_op = update_docs_collection->update_matching_filter("user_name:=fat_cat", document.dump(), dirty_values);
+ ASSERT_TRUE(update_op.ok());
+ ASSERT_EQ(2, update_op.get()["num_updated"]);
+
+ res = update_docs_collection->search("cat data", {"content"}, "", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(2, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ("slim_cat", res["hits"][i]["document"]["user_name"].get<std::string>());
+ }
+
+ // Test batching
+ res = update_docs_collection->search("dog data", {"content"}, "", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(3, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ("fast_dog", res["hits"][i]["document"]["user_name"].get<std::string>());
+ }
+
+ document["user_name"] = "lazy_dog";
+ update_op = update_docs_collection->update_matching_filter("user_name:=fast_dog", document.dump(), dirty_values, 2);
+ ASSERT_TRUE(update_op.ok());
+ ASSERT_EQ(3, update_op.get()["num_updated"]);
+
+ res = update_docs_collection->search("dog data", {"content"}, "", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(3, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ("lazy_dog", res["hits"][i]["document"]["user_name"].get<std::string>());
+ }
+
+ // Test nested fields updation
+ res = update_docs_collection->search("*", {}, "user_name:=slim_cat", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(2, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ("cat data " + std::to_string(i + 1), res["hits"][i]["document"]["content"]["title"].get<std::string>());
+ }
+
+ document.clear();
+ document["content"]["title"] = "fancy cat title";
+
+ update_op = update_docs_collection->update_matching_filter("user_name:=slim_cat", document.dump(), dirty_values, 2);
+ ASSERT_TRUE(update_op.ok());
+ ASSERT_EQ(2, update_op.get()["num_updated"]);
+
+ res = update_docs_collection->search("*", {}, "user_name:=slim_cat", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(2, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ("fancy cat title", res["hits"][i]["document"]["content"]["title"].get<std::string>());
+ }
+
+ // Test all document updation
+ res = update_docs_collection->search("*", {}, "", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(5, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_NE(0, res["hits"][i]["document"]["likes"].get<int>());
+ }
+
+ document.clear();
+ document["likes"] = 0;
+
+ update_op = update_docs_collection->update_matching_filter("*", document.dump(), dirty_values, 2);
+ ASSERT_TRUE(update_op.ok());
+ ASSERT_EQ(5, update_op.get()["num_updated"]);
+
+ res = update_docs_collection->search("*", {}, "", {}, sort_fields, {0}, 10).get();
+ ASSERT_EQ(5, res["hits"].size());
+ for (size_t i = 0; i < res["hits"].size(); i++) {
+ ASSERT_EQ(0, res["hits"][i]["document"]["likes"].get<int>());
+ }
+
+ collectionManager.drop_collection("update_docs_collection");
+}
+
TEST_F(CollectionTest, UpdateDocumentSorting) {
Collection *coll1;