Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Check length of filter token index during prefix/exact match.
Kishore Nallan423577c04acd
diff --git a/src/posting_list.cpp b/src/posting_list.cppindex a548f9ce..e9bb2610 100644--- a/src/posting_list.cpp+++ b/src/posting_list.cpp@@ -1197,7 +1197,7 @@ void posting_list_t::get_prefix_matches(std::vector<iterator_t>& its, const bool// field is an arraystruct token_index_meta_t {- std::bitset<32> token_index;+ std::bitset<128> token_index;};for (size_t i = 0; i < num_ids; i++) {@@ -1236,7 +1236,7 @@ void posting_list_t::get_prefix_matches(std::vector<iterator_t>& its, const boolif (pos == prev_pos) { // indicates end of array indexsize_t array_index = (size_t) offsets[start_offset_index];- if (found_matching_index) {+ if (found_matching_index && j+1 < 128) {array_index_to_token_index[array_index].token_index.set(j+1);}@@ -1360,7 +1360,7 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool// field is an arraystruct token_index_meta_t {- std::bitset<32> token_index;+ std::bitset<128> token_index;bool has_last_token;};@@ -1411,7 +1411,7 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool}}- if(found_matching_index) {+ if(found_matching_index && j+1 < 128) {array_index_to_token_index[array_index].token_index.set(j+1);}@@ -1508,7 +1508,7 @@ bool posting_list_t::has_prefix_match(std::vector<posting_list_t::iterator_t>& p// field is an arraystruct token_index_meta_t {- std::bitset<32> token_index;+ std::bitset<128> token_index;};std::map<size_t, token_index_meta_t> array_index_to_token_index;@@ -1540,7 +1540,7 @@ bool posting_list_t::has_prefix_match(std::vector<posting_list_t::iterator_t>& pif (pos == prev_pos) { // indicates end of array indexsize_t array_index = (size_t) offsets[start_offset_index];- if (found_matching_index) {+ if (found_matching_index && i+1 < 128) {array_index_to_token_index[array_index].token_index.set(i + 1);}@@ -1630,7 +1630,7 @@ bool posting_list_t::has_exact_match(std::vector<posting_list_t::iterator_t>& po// field is an arraystruct token_index_meta_t {- std::bitset<32> token_index;+ std::bitset<128> token_index;bool has_last_token;};@@ -1674,7 +1674,7 @@ bool posting_list_t::has_exact_match(std::vector<posting_list_t::iterator_t>& po}}- if(found_matching_index) {+ if(found_matching_index && i+1 < 128) {array_index_to_token_index[array_index].token_index.set(i + 1);}diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex e704cec7..128f64c2 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -2927,6 +2927,36 @@ TEST_F(CollectionFilteringTest, PrefixFilterOnTextFields) {ASSERT_EQ("Steve Reiley", res_obj["hits"][3]["document"]["names"][0]);}+TEST_F(CollectionFilteringTest, ExactFilterOnLongField) {+ nlohmann::json schema = R"({+ "name": "companies",+ "fields": [+ {"name": "keywords", "type": "string[]"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());++ auto coll = op.get();++ nlohmann::json doc1;+ doc1["id"] = "0";++ std::string arr_value;++ // when value exceeds 128 tokens, we will fail gracefully+ for(size_t i = 0; i < 130; i++) {+ arr_value += "foo" + std::to_string(i) + " ";+ }+ doc1["keywords"] = {arr_value};++ ASSERT_TRUE(coll->add(doc1.dump()).ok());++ auto results = coll->search("*", {}, "keywords:=" + arr_value, {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(0, results["hits"].size());+}+TEST_F(CollectionFilteringTest, FilterOnStemmedField) {nlohmann::json schema = R"({"name": "companies",