Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Some skeleton doc operations like incrementing.

Kishore Nallan3e1af0ed246b
diff --git a/include/index.h b/include/index.h
index 8a622960..38fdc576 100644
--- a/include/index.h
+++ b/include/index.h
@@ -525,8 +525,12 @@ private:
static void compute_facet_stats(facet &a_facet, uint64_t raw_value, const std::string & field_type);
- static void get_doc_changes(const index_operation_t op, nlohmann::json &update_doc,
- const nlohmann::json &old_doc, nlohmann::json &new_doc, nlohmann::json &del_doc);
+ static void handle_doc_ops(const tsl::htrie_map<char, field>& search_schema,
+ nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc);
+
+ static void get_doc_changes(const index_operation_t op, const tsl::htrie_map<char, field>& search_schema,
+ nlohmann::json &update_doc, const nlohmann::json &old_doc, nlohmann::json &new_doc,
+ nlohmann::json &del_doc);
bool common_results_exist(std::vector<art_leaf*>& leaves, bool must_match_phrase) const;
diff --git a/src/index.cpp b/src/index.cpp
index 6426b4ee..b337bb4b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -450,8 +450,8 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
if(index_rec.is_update) {
// scrub string fields to reduce delete ops
- get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,
- index_rec.del_doc);
+ get_doc_changes(index_rec.operation, search_schema, index_rec.doc, index_rec.old_doc,
+ index_rec.new_doc, index_rec.del_doc);
scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
embed_fields(index_rec.new_doc, embedding_fields, search_schema);
} else {
@@ -6129,13 +6129,53 @@ void Index::refresh_schemas(const std::vector<field>& new_fields, const std::vec
}
}
-void Index::get_doc_changes(const index_operation_t op, nlohmann::json& update_doc,
- const nlohmann::json& old_doc, nlohmann::json& new_doc, nlohmann::json& del_doc) {
+void Index::handle_doc_ops(const tsl::htrie_map<char, field>& search_schema,
+ nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc) {
+
+ /*
+ {
+ "$operations": {
+ "increment": {"likes": 1, "views": 20}
+ }
+ }
+ */
+
+ auto ops_it = update_doc.find("$operations");
+ if(ops_it != update_doc.end()) {
+ const auto& operations = ops_it.value();
+ if(operations.contains("increment") && operations["increment"].is_object()) {
+ for(const auto& item: operations["increment"].items()) {
+ auto field_it = search_schema.find(item.key());
+ if(field_it != search_schema.end()) {
+ if(field_it->type == field_types::INT32 && item.value().is_number_integer()) {
+ int32_t existing_value = 0;
+ if(old_doc.contains(item.key())) {
+ existing_value = old_doc[item.key()].get<int32_t>();
+ }
+
+ auto updated_value = existing_value + item.value().get<int32>();
+ new_doc[item.key()] = updated_value;
+ update_doc[item.key()] = updated_value;
+ }
+ }
+ }
+ }
+
+ update_doc.erase("$operations");
+ }
+}
+
+void Index::get_doc_changes(const index_operation_t op, const tsl::htrie_map<char, field>& search_schema,
+ nlohmann::json& update_doc, const nlohmann::json& old_doc, nlohmann::json& new_doc,
+ nlohmann::json& del_doc) {
if(op == UPSERT) {
new_doc = update_doc;
} else {
new_doc = old_doc;
+
+ handle_doc_ops(search_schema, update_doc, old_doc, new_doc);
+
new_doc.merge_patch(update_doc);
if(old_doc.contains(".flat")) {
new_doc[".flat"] = old_doc[".flat"];
diff --git a/test/collection_operations_test.cpp b/test/collection_operations_test.cpp
new file mode 100644
index 00000000..e736b226
--- /dev/null
+++ b/test/collection_operations_test.cpp
@@ -0,0 +1,93 @@
+#include <gtest/gtest.h>
+#include <string>
+#include <vector>
+#include <fstream>
+#include <algorithm>
+#include <collection_manager.h>
+#include "collection.h"
+
+class CollectionOperationsTest : public ::testing::Test {
+protected:
+ Store *store;
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
+
+ std::vector<std::string> query_fields;
+ std::vector<sort_by> sort_fields;
+
+ void setupCollection() {
+ std::string state_dir_path = "/tmp/typesense_test/collection_operations";
+ LOG(INFO) << "Truncating and creating: " << state_dir_path;
+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
+
+ store = new Store(state_dir_path);
+ collectionManager.init(store, 1.0, "auth_key", quit);
+ collectionManager.load(8, 1000);
+ }
+
+ virtual void SetUp() {
+ setupCollection();
+ }
+
+ virtual void TearDown() {
+ collectionManager.dispose();
+ delete store;
+ }
+};
+
+TEST_F(CollectionOperationsTest, IncrementInt32Value) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"},
+ {"name": "points", "type": "int32"}
+ ]
+ })"_json;
+
+ Collection *coll = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "Sherlock Holmes";
+ doc["points"] = 100;
+ ASSERT_TRUE(coll->add(doc.dump()).ok());
+
+ // increment by 1
+ doc["id"] = "0";
+ doc["$operations"] = R"({"increment": {"points": 1}})"_json;
+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());
+
+ auto res = coll->search("*", {"title"}, "points:101", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, res["hits"].size());
+
+ ASSERT_EQ(3, res["hits"][0]["document"].size());
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("Sherlock Holmes", res["hits"][0]["document"]["title"].get<std::string>());
+ ASSERT_EQ(101, res["hits"][0]["document"]["points"].get<size_t>());
+
+ // increment by 10
+ doc["id"] = "0";
+ doc["$operations"] = R"({"increment": {"points": 10}})"_json;
+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());
+
+ res = coll->search("*", {"title"}, "points:111", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(3, res["hits"][0]["document"].size());
+
+ // decrement by 10 using negative number
+ doc["id"] = "0";
+ doc["$operations"] = R"({"increment": {"points": -10}})"_json;
+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());
+
+ res = coll->search("*", {"title"}, "points:101", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(3, res["hits"][0]["document"].size());
+
+ // bad field - should not change anything
+ doc["id"] = "0";
+ doc["$operations"] = R"({"increment": {"pointsx": -10}})"_json;
+ ASSERT_TRUE(coll->add(doc.dump(), UPDATE).ok());
+ res = coll->search("*", {"title"}, "points:101", {}, {}, {0}, 3, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(3, res["hits"][0]["document"].size());
+}
\ No newline at end of file