Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Some skeleton doc operations like incrementing.
Kishore Nallan3e1af0ed246b
diff --git a/include/index.h b/include/index.hindex 8a622960..38fdc576 100644--- a/include/index.h+++ b/include/index.h@@ -525,8 +525,12 @@ private:static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);- static void get_doc_changes(const index_operation_t op, nlohmann::json &update_doc,- const nlohmann::json &old_doc, nlohmann::json &new_doc, nlohmann::json &del_doc);+ static void handle_doc_ops(const tsl::htrie_map<char, field>& search_schema,+ nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc);++ static void get_doc_changes(const index_operation_t op, const tsl::htrie_map<char, field>& search_schema,+ nlohmann::json &update_doc, const nlohmann::json &old_doc, nlohmann::json &new_doc,+ nlohmann::json &del_doc);bool common_results_exist(std::vector<art_leaf*>& leaves, bool must_match_phrase) const;diff --git a/src/index.cpp b/src/index.cppindex 6426b4ee..b337bb4b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -450,8 +450,8 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteif(index_rec.is_update) {// scrub string fields to reduce delete ops- get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,- index_rec.del_doc);+ get_doc_changes(index_rec.operation, search_schema, index_rec.doc, index_rec.old_doc,+ index_rec.new_doc, index_rec.del_doc);scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);embed_fields(index_rec.new_doc, embedding_fields, search_schema);} else {@@ -6129,13 +6129,53 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec}}-void Index::get_doc_changes(const index_operation_t op, nlohmann::json& update_doc,- const nlohmann::json& old_doc, nlohmann::json& new_doc, nlohmann::json& del_doc) {+void Index::handle_doc_ops(const tsl::htrie_map<char, field>& search_schema,+ nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc) {++ /*+ {+ "$operations": {+ "increment": {"likes": 1, "views": 20}+ }+ }+ */++ auto ops_it = update_doc.find("$operations");+ if(ops_it != update_doc.end()) {+ const auto& operations = ops_it.value();+ if(operations.contains("increment") && operations["increment"].is_object()) {+ for(const auto& item: operations["increment"].items()) {+ auto field_it = search_schema.find(item.key());+ if(field_it != search_schema.end()) {+ if(field_it->type == field_types::INT32 && item.value().is_number_integer()) {+ int32_t existing_value = 0;+ if(old_doc.contains(item.key())) {+ existing_value = old_doc[item.key()].get<int32_t>();+ }++ auto updated_value = existing_value + item.value().get<int32>();+ new_doc[item.key()] = updated_value;+ update_doc[item.key()] = updated_value;+ }+ }+ }+ }++ update_doc.erase("$operations");+ }+}++void Index::get_doc_changes(const index_operation_t op, const tsl::htrie_map<char, field>& search_schema,+ nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc,+ nlohmann::json& del_doc) {if(op == UPSERT) {new_doc = update_doc;} else {new_doc = old_doc;++ handle_doc_ops(search_schema, update_doc, old_doc, new_doc);+new_doc.merge_patch(update_doc);if(old_doc.contains(".flat")) {new_doc[".flat"] = old_doc[".flat"];diff --git a/test/collection_operations_test.cpp b/test/collection_operations_test.cppnew file mode 100644index 00000000..e736b226--- /dev/null+++ b/test/collection_operations_test.cpp@@ -0,0 +1,93 @@+#include <gtest/gtest.h>+#include <string>+#include <vector>+#include <fstream>+#include <algorithm>+#include <collection_manager.h>+#include "collection.h"++class CollectionOperationsTest : public ::testing::Test {+protected:+ Store *store;+ CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;++ std::vector<std::string> query_fields;+ std::vector<sort_by> sort_fields;++ void setupCollection() {+ std::string state_dir_path = "/tmp/typesense_test/collection_operations";+ LOG(INFO) << "Truncating and creating: " << state_dir_path;+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);+ }++ virtual void SetUp() {+ setupCollection();+ }++ virtual void TearDown() {+ collectionManager.dispose();+ delete store;+ }+};++TEST_F(CollectionOperationsTest, IncrementInt32Value) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"},+ {"name": "points", "type": "int32"}+ ]+ })"_json;++ Collection *coll = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["title"] = "Sherlock Holmes";+ doc["points"] = 100;+ ASSERT_TRUE(coll->add(doc.dump()).ok());++ // increment by 1+ doc["id"] = "0";+ doc["$operations"] = R"({"increment": {"points": 1}})"_json;+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());++ auto res = coll->search("*", {"title"}, "points:101", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, res["hits"].size());++ ASSERT_EQ(3, res["hits"][0]["document"].size());+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("Sherlock Holmes", res["hits"][0]["document"]["title"].get<std::string>());+ ASSERT_EQ(101, res["hits"][0]["document"]["points"].get<size_t>());++ // increment by 10+ doc["id"] = "0";+ doc["$operations"] = R"({"increment": {"points": 10}})"_json;+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());++ res = coll->search("*", {"title"}, "points:111", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(3, res["hits"][0]["document"].size());++ // decrement by 10 using negative number+ doc["id"] = "0";+ doc["$operations"] = R"({"increment": {"points": -10}})"_json;+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());++ res = coll->search("*", {"title"}, "points:101", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(3, res["hits"][0]["document"].size());++ // bad field - should not change anything+ doc["id"] = "0";+ doc["$operations"] = R"({"increment": {"pointsx": -10}})"_json;+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());+ res = coll->search("*", {"title"}, "points:101", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(3, res["hits"][0]["document"].size());+}\ No newline at end of file