Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Trim down filter token to 100 characters.
Harpreet Sangar31ae2ce0703b
diff --git a/src/index.cpp b/src/index.cppindex 98e1eb56..f77bf06a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1862,6 +1862,9 @@ Option<bool> Index::do_filtering(filter_node_t* const root,std::vector<std::string> str_tokens;while (tokenizer.next(str_token, token_index)) {+ if (str_token.size() > 100) {+ str_token.erase(100);+ }str_tokens.push_back(str_token);art_leaf* leaf = (art_leaf *) art_search(t, (const unsigned char*) str_token.c_str(),diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex 0e736e23..cba5b21d 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -2650,4 +2650,43 @@ TEST_F(CollectionFilteringTest, ComplexFilterQuery) {ASSERT_EQ("`filter_by` has too many operations.", search_op.error());collectionManager.drop_collection("ComplexFilterQueryCollection");-}\ No newline at end of file+}++TEST_F(CollectionFilteringTest, LargeFilterToken) {+ nlohmann::json json =+ R"({+ "name": "LargeFilterTokenCollection",+ "fields": [+ {"name": "uri", "type": "string"}+ ],+ "symbols_to_index": [+ "/",+ "-"+ ]+ })"_json;++ auto op = collectionManager.create_collection(json);+ ASSERT_TRUE(op.ok());+ auto coll = op.get();++ json.clear();+ std::string token = "rade/aols/insolvenzrecht/persoenliche-risiken-fuer-organe-von-kapitalgesellschaften-gmbh-"+ "geschaeftsfuehrer-ag-vorstand";+ json["uri"] = token;+ auto add_op = coll->add(json.dump());+ ASSERT_TRUE(add_op.ok());++ auto results = coll->search("*", query_fields, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());++ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());++ token.erase(100); // Max token length that's indexed is 100, we'll still get a match.+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());++ token.erase(99);+ results = coll->search("*", query_fields, "uri:" + token, {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(0, results["hits"].size());+}