Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Account for token separator during upsert/removal.

Kishore Nallan8eb19f70a57a
diff --git a/include/index.h b/include/index.h
index a001dec2..27aa6a16 100644
--- a/include/index.h
+++ b/include/index.h
@@ -749,7 +749,9 @@ public:
static void tokenize_string_field(const nlohmann::json& document,
const field& search_field, std::vector<std::string>& tokens,
- const std::string& locale);
+ const std::string& locale,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators);
// Public operations
diff --git a/src/index.cpp b/src/index.cpp
index fd000a3d..e02d33d7 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -4338,7 +4338,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const
// Go through all the field names and find the keys+values so that they can be removed from in-memory index
if(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {
std::vector<std::string> tokens;
- tokenize_string_field(document, search_field, tokens, search_field.locale);
+ tokenize_string_field(document, search_field, tokens, search_field.locale, symbols_to_index, token_separators);
for(size_t i = 0; i < tokens.size(); i++) {
const auto& token = tokens[i];
@@ -4477,16 +4477,18 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc
}
void Index::tokenize_string_field(const nlohmann::json& document, const field& search_field,
- std::vector<std::string>& tokens, const std::string& locale) {
+ std::vector<std::string>& tokens, const std::string& locale,
+ const std::vector<char>& symbols_to_index,
+ const std::vector<char>& token_separators) {
const std::string& field_name = search_field.name;
if(search_field.type == field_types::STRING) {
- Tokenizer(document[field_name], true, false, locale).tokenize(tokens);
+ Tokenizer(document[field_name], true, false, locale, symbols_to_index, token_separators).tokenize(tokens);
} else if(search_field.type == field_types::STRING_ARRAY) {
const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();
for(const std::string & value: values) {
- Tokenizer(value, true, false, locale).tokenize(tokens);
+ Tokenizer(value, true, false, locale, symbols_to_index, token_separators).tokenize(tokens);
}
}
}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index 59a887b9..ed2486de 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -2293,17 +2293,59 @@ TEST_F(CollectionFilteringTest, ExcludeMultipleTokens) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArray) {
+TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArrayWithTokenSeparators) {
std::vector<field> fields = {field("name", field_types::STRING, false), field("tags", field_types::STRING_ARRAY, false)};
- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, "").get();
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, "", {}, {"-"}).get();
nlohmann::json doc1;
doc1["id"] = "0";
doc1["name"] = "david";
doc1["tags"] = {"alpha-beta-gamma", "foo-bar-baz"};
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "david";
+ doc2["tags"] = {"alpha-gamma-beta", "bar-foo-baz"};
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // upsert with "foo-bar-baz" removed
+ doc1["tags"] = {"alpha-beta-gamma"};
+ coll1->add(doc1.dump(), UPSERT);
+
+ results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search("david", {"name"},"tags:=[bar-foo-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArrayWithSymbolsToIndex) {
+ std::vector<field> fields = {field("name", field_types::STRING, false), field("tags", field_types::STRING_ARRAY, false)};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, "", {"-"}, {}).get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["name"] = "david";
+ doc1["tags"] = {"alpha-beta-gamma", "foo-bar-baz"};
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["name"] = "david";
+ doc2["tags"] = {"alpha-gamma-beta", "bar-foo-baz"};
+
ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
auto results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(1, results["hits"].size());
@@ -2316,5 +2358,9 @@ TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArray) {
results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(0, results["hits"].size());
+ results = coll1->search("david", {"name"},"tags:=[bar-foo-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+
collectionManager.drop_collection("coll1");
}