Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle infix update/deletion.

Kishore Nallan9e4d51cfcc76
diff --git a/include/index.h b/include/index.h
index 6de03cb5..d3d47b9e 100644
--- a/include/index.h
+++ b/include/index.h
@@ -668,6 +668,8 @@ public:
const spp::sparse_hash_map<std::string, num_tree_t*>& _get_numerical_index() const;
+ const spp::sparse_hash_map<std::string, array_mapped_infix_t>& _get_infix_index() const;
+
static int get_bounded_typo_cost(const size_t max_cost, const size_t token_len,
size_t min_len_1typo, size_t min_len_2typo);
diff --git a/src/index.cpp b/src/index.cpp
index 7f2baea8..dffb6a37 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3740,6 +3740,12 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
posting_t::destroy_list(values);
}
}
+
+ if(search_field.infix) {
+ auto strhash = StringUtils::hash_wy(key, token.size());
+ const auto& infix_sets = infix_index.at(search_field.name);
+ infix_sets[strhash % 4]->erase(token);
+ }
}
} else if(search_field.is_int32()) {
const std::vector<int32_t>& values = search_field.is_single_integer() ?
@@ -3867,6 +3873,10 @@ const spp::sparse_hash_map<std::string, num_tree_t*>& Index::_get_numerical_inde
return numerical_index;
}
+const spp::sparse_hash_map<std::string, array_mapped_infix_t>& Index::_get_infix_index() const {
+ return infix_index;
+};
+
void Index::refresh_schemas(const std::vector<field>& new_fields) {
std::unique_lock lock(mutex);
diff --git a/test/collection_infix_search_test.cpp b/test/collection_infix_search_test.cpp
index 0eb36e5c..b20f8271 100644
--- a/test/collection_infix_search_test.cpp
+++ b/test/collection_infix_search_test.cpp
@@ -248,3 +248,93 @@ TEST_F(CollectionInfixSearchTest, InfixSpecificField) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionInfixSearchTest, InfixDeleteAndUpdate) {
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "", -1, 1),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "GH100037IN8900X";
+ doc["points"] = 100;
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("100037",
+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ coll1->remove("0");
+
+ for(size_t i = 0; i < coll1->_get_index()->_get_infix_index().at("title").size(); i++) {
+ ASSERT_EQ(0, coll1->_get_index()->_get_infix_index().at("title").at(i)->size());
+ }
+
+ results = coll1->search("100037",
+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(0, results["found"].get<size_t>());
+ ASSERT_EQ(0, results["hits"].size());
+
+ // add the document again and then update it
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ results = coll1->search("100037",
+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+
+ doc["title"] = "YHD3342D78912";
+ ASSERT_TRUE(coll1->add(doc.dump(), UPSERT).ok());
+
+ results = coll1->search("342D78",
+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ results = coll1->search("100037",
+ {"title"}, "", {}, {}, {0}, 3, 1, FREQUENCY, {true}, 5,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, true,
+ 4, {always}).get();
+
+ ASSERT_EQ(0, results["found"].get<size_t>());
+ ASSERT_EQ(0, results["hits"].size());
+
+ std::string key = "yhd3342d78912";
+ auto strhash = StringUtils::hash_wy(key.c_str(), key.size());
+ const auto& infix_sets = coll1->_get_index()->_get_infix_index().at("title");
+ ASSERT_EQ(1, infix_sets[strhash % 4]->size());
+
+ for(size_t i = 0; i < infix_sets.size(); i++) {
+ if(i != strhash % 4) {
+ ASSERT_EQ(0, infix_sets[i]->size());
+ }
+ }
+
+ collectionManager.drop_collection("coll1");
+}