Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Update documents using `filter_by`. (#845)
Harpreet Sangar5f034f8b3b3f
diff --git a/include/collection.h b/include/collection.hindex 6b3d9e71..7017098f 100644--- a/include/collection.h+++ b/include/collection.h@@ -370,6 +370,11 @@ public:const DIRTY_VALUES& dirty_values=DIRTY_VALUES::COERCE_OR_REJECT,const bool& return_doc=false, const bool& return_id=false);+ Option<nlohmann::json> update_matching_filter(const std::string& filter_query,+ const std::string & json_str,+ std::string& req_dirty_values,+ const int batch_size = 1000);+Option<nlohmann::json> search(const std::string & query, const std::vector<std::string> & search_fields,const std::string & filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,@@ -414,8 +419,8 @@ public:const size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0) const;- Option<bool> get_filter_ids(const std::string & simple_filter_query,- std::vector<std::pair<size_t, uint32_t*>>& index_ids);+ Option<bool> get_filter_ids(const std::string & filter_query,+ std::vector<std::pair<size_t, uint32_t*>>& index_ids) const;Option<nlohmann::json> get(const std::string & id) const;diff --git a/include/core_api.h b/include/core_api.hindex 8130434c..0bc1a615 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -33,6 +33,8 @@ bool post_add_document(const std::shared_ptr<http_req>& req, const std::shared_pbool patch_update_document(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool patch_update_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool post_import_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);bool get_fetch_document(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/include/field.h b/include/field.hindex b8dbca82..242f8f9b 100644--- a/include/field.h+++ b/include/field.h@@ -513,7 +513,7 @@ struct filter {std::string& processed_filter_val,NUM_COMPARATOR& num_comparator);- static Option<bool> parse_filter_query(const std::string& simple_filter_query,+ static Option<bool> parse_filter_query(const std::string& filter_query,const tsl::htrie_map<char, field>& search_schema,const Store* store,const std::string& doc_id_prefix,diff --git a/src/collection.cpp b/src/collection.cppindex b6ef3405..ae30fbf3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -322,6 +322,100 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmareturn resp_summary;}+Option<nlohmann::json> Collection::update_matching_filter(const std::string& filter_query,+ const std::string & json_str,+ std::string& req_dirty_values,+ const int batch_size) {+ auto _filter_query = filter_query;+ StringUtils::trim(_filter_query);++ if (_filter_query.empty()) {+ nlohmann::json resp_summary;+ resp_summary["num_updated"] = 0;+ return Option(resp_summary);+ }++ const auto& dirty_values = parse_dirty_values_option(req_dirty_values);+ size_t docs_updated_count;+ nlohmann::json update_document, dummy;++ try {+ update_document = nlohmann::json::parse(json_str);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ return Option<nlohmann::json>(400, std::string("Bad JSON: ") + e.what());+ }++ std::vector<std::string> buffer;+ buffer.reserve(batch_size);++ if (_filter_query == "*") {+ // Get an iterator from rocksdb and iterate over all the documents present in the collection.+ std::string iter_upper_bound_key = get_seq_id_collection_prefix() + "`";+ auto iter_upper_bound = new rocksdb::Slice(iter_upper_bound_key);+ CollectionManager & collectionManager = CollectionManager::get_instance();+ const std::string seq_id_prefix = get_seq_id_collection_prefix();+ rocksdb::Iterator* it = collectionManager.get_store()->scan(seq_id_prefix, iter_upper_bound);++ while(it->Valid()) {+ // Generate a batch of documents to be ingested by add_many.+ for (int buffer_counter = 0; buffer_counter < batch_size && it->Valid();) {+ auto json_doc_str = it->value().ToString();+ it->Next();+ nlohmann::json existing_document;+ try {+ existing_document = nlohmann::json::parse(json_doc_str);+ } catch(...) {+ continue; // Don't add into buffer.+ }++ update_document["id"] = existing_document["id"].get<std::string>();+ buffer.push_back(update_document.dump());+ buffer_counter++;+ }++ auto res = add_many(buffer, dummy, index_operation_t::UPDATE, "", dirty_values);+ docs_updated_count += res["num_imported"].get<size_t>();+ buffer.clear();+ }++ delete iter_upper_bound;+ delete it;+ } else {+ std::vector<std::pair<size_t, uint32_t*>> filter_ids;+ auto filter_ids_op = get_filter_ids(_filter_query, filter_ids);+ if(!filter_ids_op.ok()) {+ return Option<nlohmann::json>(filter_ids_op.code(), filter_ids_op.error());+ }++ for (size_t i = 0; i < filter_ids[0].first;) {+ for (int buffer_counter = 0; buffer_counter < batch_size && i < filter_ids[0].first;) {+ uint32_t seq_id = *(filter_ids[0].second + i++);+ nlohmann::json existing_document;++ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), existing_document);+ if (!get_doc_op.ok()) {+ continue;+ }++ update_document["id"] = existing_document["id"].get<std::string>();+ buffer.push_back(update_document.dump());+ buffer_counter++;+ }++ auto res = add_many(buffer, dummy, index_operation_t::UPDATE, "", dirty_values);+ docs_updated_count += res["num_imported"].get<size_t>();+ buffer.clear();+ }++ delete [] filter_ids[0].second;+ }++ nlohmann::json resp_summary;+ resp_summary["num_updated"] = docs_updated_count;+ return Option(resp_summary);+}+bool Collection::is_exceeding_memory_threshold() const {return SystemMetrics::used_memory_ratio() > max_memory_ratio;}@@ -2225,13 +2319,13 @@ void Collection::populate_result_kvs(Topster *topster, std::vector<std::vector<K}}-Option<bool> Collection::get_filter_ids(const std::string & simple_filter_query,- std::vector<std::pair<size_t, uint32_t*>>& index_ids) {+Option<bool> Collection::get_filter_ids(const std::string & filter_query,+ std::vector<std::pair<size_t, uint32_t*>>& index_ids) const {std::shared_lock lock(mutex);const std::string doc_id_prefix = std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_";filter_node_t* filter_tree_root = nullptr;- Option<bool> filter_op = filter::parse_filter_query(simple_filter_query, search_schema,+ Option<bool> filter_op = filter::parse_filter_query(filter_query, search_schema,store, doc_id_prefix, filter_tree_root);if(!filter_op.ok()) {diff --git a/src/core_api.cpp b/src/core_api.cppindex 1599ddd2..09060a1e 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -944,6 +944,38 @@ bool patch_update_document(const std::shared_ptr<http_req>& req, const std::sharreturn true;}+bool patch_update_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ const char *FILTER_BY = "filter_by";+ std::string filter_query;+ if(req->params.count(FILTER_BY) == 0) {+ res->set_400("Parameter `" + std::string(FILTER_BY) + "` must be provided.");+ return false;+ } else {+ filter_query = req->params[FILTER_BY];+ }++ CollectionManager & collectionManager = CollectionManager::get_instance();+ auto collection = collectionManager.get_collection(req->params["collection"]);+ if(collection == nullptr) {+ res->set_404();+ return false;+ }++ const char* DIRTY_VALUES_PARAM = "dirty_values";+ if(req->params.count(DIRTY_VALUES_PARAM) == 0) {+ req->params[DIRTY_VALUES_PARAM] = ""; // set it empty as default will depend on whether schema is enabled+ }++ auto update_op = collection->update_matching_filter(filter_query, req->body, req->params[DIRTY_VALUES_PARAM]);+ if(update_op.ok()) {+ res->set_200(update_op.get().dump());+ } else {+ res->set(update_op.code(), update_op.error());+ }++ return update_op.ok();+}+bool get_fetch_document(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {std::string doc_id = req->params["id"];diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 79479ab7..b32057ed 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -30,6 +30,7 @@ void master_server_routes() {server->get("/collections/:collection/documents/:id", get_fetch_document);server->patch("/collections/:collection/documents/:id", patch_update_document);+ server->patch("/collections/:collection/documents", patch_update_documents);server->del("/collections/:collection/documents/:id", del_remove_document);server->get("/collections/:collection/overrides", get_overrides);diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 9aa327d9..2e547041 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2528,6 +2528,124 @@ TEST_F(CollectionTest, UpdateDocument) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionTest, UpdateDocuments) {+ nlohmann::json schema = R"({+ "name": "update_docs_collection",+ "enable_nested_fields": true,+ "fields": [+ {"name": "user_name", "type": "string", "facet": true},+ {"name": "likes", "type": "int32"},+ {"name": "content", "type": "object"}+ ],+ "default_sorting_field": "likes"+ })"_json;++ Collection *update_docs_collection = collectionManager.get_collection("update_docs_collection").get();+ if (update_docs_collection == nullptr) {+ auto op = CollectionManager::create_collection(schema);+ ASSERT_TRUE(op.ok());+ update_docs_collection = op.get();+ }++ std::vector<std::string> json_lines = {+ R"({"user_name": "fat_cat","likes": 5215,"content": {"title": "cat data 1", "body": "cd1"}})",+ R"({"user_name": "fast_dog","likes": 273,"content": {"title": "dog data 1", "body": "dd1"}})",+ R"({"user_name": "fat_cat","likes": 2133,"content": {"title": "cat data 2", "body": "cd2"}})",+ R"({"user_name": "fast_dog","likes": 9754,"content": {"title": "dog data 2", "body": "dd2"}})",+ R"({"user_name": "fast_dog","likes": 576,"content": {"title": "dog data 3", "body": "dd3"}})"+ };++ for (auto const& json: json_lines){+ auto add_op = update_docs_collection->add(json);+ if (!add_op.ok()) {+ std::cout << add_op.error() << std::endl;+ }+ ASSERT_TRUE(add_op.ok());+ }++ std::vector<sort_by> sort_fields = { sort_by("likes", "DESC") };++ auto res = update_docs_collection->search("cat data", {"content"}, "", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(2, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ("fat_cat", res["hits"][i]["document"]["user_name"].get<std::string>());+ }++ nlohmann::json document;+ document["user_name"] = "slim_cat";+ std::string dirty_values;++ auto update_op = update_docs_collection->update_matching_filter("user_name:=fat_cat", document.dump(), dirty_values);+ ASSERT_TRUE(update_op.ok());+ ASSERT_EQ(2, update_op.get()["num_updated"]);++ res = update_docs_collection->search("cat data", {"content"}, "", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(2, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ("slim_cat", res["hits"][i]["document"]["user_name"].get<std::string>());+ }++ // Test batching+ res = update_docs_collection->search("dog data", {"content"}, "", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(3, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ("fast_dog", res["hits"][i]["document"]["user_name"].get<std::string>());+ }++ document["user_name"] = "lazy_dog";+ update_op = update_docs_collection->update_matching_filter("user_name:=fast_dog", document.dump(), dirty_values, 2);+ ASSERT_TRUE(update_op.ok());+ ASSERT_EQ(3, update_op.get()["num_updated"]);++ res = update_docs_collection->search("dog data", {"content"}, "", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(3, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ("lazy_dog", res["hits"][i]["document"]["user_name"].get<std::string>());+ }++ // Test nested fields updation+ res = update_docs_collection->search("*", {}, "user_name:=slim_cat", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(2, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ("cat data " + std::to_string(i + 1), res["hits"][i]["document"]["content"]["title"].get<std::string>());+ }++ document.clear();+ document["content"]["title"] = "fancy cat title";++ update_op = update_docs_collection->update_matching_filter("user_name:=slim_cat", document.dump(), dirty_values, 2);+ ASSERT_TRUE(update_op.ok());+ ASSERT_EQ(2, update_op.get()["num_updated"]);++ res = update_docs_collection->search("*", {}, "user_name:=slim_cat", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(2, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ("fancy cat title", res["hits"][i]["document"]["content"]["title"].get<std::string>());+ }++ // Test all document updation+ res = update_docs_collection->search("*", {}, "", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(5, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_NE(0, res["hits"][i]["document"]["likes"].get<int>());+ }++ document.clear();+ document["likes"] = 0;++ update_op = update_docs_collection->update_matching_filter("*", document.dump(), dirty_values, 2);+ ASSERT_TRUE(update_op.ok());+ ASSERT_EQ(5, update_op.get()["num_updated"]);++ res = update_docs_collection->search("*", {}, "", {}, sort_fields, {0}, 10).get();+ ASSERT_EQ(5, res["hits"].size());+ for (size_t i = 0; i < res["hits"].size(); i++) {+ ASSERT_EQ(0, res["hits"][i]["document"]["likes"].get<int>());+ }++ collectionManager.drop_collection("update_docs_collection");+}+TEST_F(CollectionTest, UpdateDocumentSorting) {Collection *coll1;