Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Account for token separator during upsert/removal.
Kishore Nallan8eb19f70a57a
diff --git a/include/index.h b/include/index.hindex a001dec2..27aa6a16 100644--- a/include/index.h+++ b/include/index.h@@ -749,7 +749,9 @@ public:static void tokenize_string_field(const nlohmann::json& document,const field& search_field, std::vector<std::string>& tokens,- const std::string& locale);+ const std::string& locale,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators);// Public operationsdiff --git a/src/index.cpp b/src/index.cppindex fd000a3d..e02d33d7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4338,7 +4338,7 @@ void Index::remove_field(uint32_t seq_id, const nlohmann::json& document, const// Go through all the field names and find the keys+values so that they can be removed from in-memory indexif(search_field.type == field_types::STRING_ARRAY || search_field.type == field_types::STRING) {std::vector<std::string> tokens;- tokenize_string_field(document, search_field, tokens, search_field.locale);+ tokenize_string_field(document, search_field, tokens, search_field.locale, symbols_to_index, token_separators);for(size_t i = 0; i < tokens.size(); i++) {const auto& token = tokens[i];@@ -4477,16 +4477,18 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & doc}void Index::tokenize_string_field(const nlohmann::json& document, const field& search_field,- std::vector<std::string>& tokens, const std::string& locale) {+ std::vector<std::string>& tokens, const std::string& locale,+ const std::vector<char>& symbols_to_index,+ const std::vector<char>& token_separators) {const std::string& field_name = search_field.name;if(search_field.type == field_types::STRING) {- Tokenizer(document[field_name], true, false, locale).tokenize(tokens);+ Tokenizer(document[field_name], true, false, locale, symbols_to_index, token_separators).tokenize(tokens);} else if(search_field.type == field_types::STRING_ARRAY) {const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();for(const std::string & value: values) {- Tokenizer(value, true, false, locale).tokenize(tokens);+ Tokenizer(value, true, false, locale, symbols_to_index, token_separators).tokenize(tokens);}}}diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 59a887b9..ed2486de 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -2293,17 +2293,59 @@ TEST_F(CollectionFilteringTest, ExcludeMultipleTokens) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArray) {+TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArrayWithTokenSeparators) {std::vector<field> fields = {field("name", field_types::STRING, false), field("tags", field_types::STRING_ARRAY, false)};- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, "").get();+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, "", {}, {"-"}).get();nlohmann::json doc1;doc1["id"] = "0";doc1["name"] = "david";doc1["tags"] = {"alpha-beta-gamma", "foo-bar-baz"};+ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "david";+ doc2["tags"] = {"alpha-gamma-beta", "bar-foo-baz"};++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ // upsert with "foo-bar-baz" removed+ doc1["tags"] = {"alpha-beta-gamma"};+ coll1->add(doc1.dump(), UPSERT);++ results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search("david", {"name"},"tags:=[bar-foo-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArrayWithSymbolsToIndex) {+ std::vector<field> fields = {field("name", field_types::STRING, false), field("tags", field_types::STRING_ARRAY, false)};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "", 0, "", {"-"}, {}).get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["name"] = "david";+ doc1["tags"] = {"alpha-beta-gamma", "foo-bar-baz"};++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["name"] = "david";+ doc2["tags"] = {"alpha-gamma-beta", "bar-foo-baz"};+ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());auto results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());@@ -2316,5 +2358,9 @@ TEST_F(CollectionFilteringTest, FilteringAfterUpsertOnArray) {results = coll1->search("david", {"name"},"tags:=[foo-bar-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(0, results["hits"].size());+ results = coll1->search("david", {"name"},"tags:=[bar-foo-baz]", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+collectionManager.drop_collection("coll1");}