Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Check length of filter token index during prefix/exact match.

Kishore Nallan423577c04acd
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index a548f9ce..e9bb2610 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1197,7 +1197,7 @@ void posting_list_t::get_prefix_matches(std::vector<iterator_t>& its, const bool
// field is an array
struct token_index_meta_t {
- std::bitset<32> token_index;
+ std::bitset<128> token_index;
};
for (size_t i = 0; i < num_ids; i++) {
@@ -1236,7 +1236,7 @@ void posting_list_t::get_prefix_matches(std::vector<iterator_t>& its, const bool
if (pos == prev_pos) { // indicates end of array index
size_t array_index = (size_t) offsets[start_offset_index];
- if (found_matching_index) {
+ if (found_matching_index && j+1 < 128) {
array_index_to_token_index[array_index].token_index.set(j+1);
}
@@ -1360,7 +1360,7 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool
// field is an array
struct token_index_meta_t {
- std::bitset<32> token_index;
+ std::bitset<128> token_index;
bool has_last_token;
};
@@ -1411,7 +1411,7 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool
}
}
- if(found_matching_index) {
+ if(found_matching_index && j+1 < 128) {
array_index_to_token_index[array_index].token_index.set(j+1);
}
@@ -1508,7 +1508,7 @@ bool posting_list_t::has_prefix_match(std::vector<posting_list_t::iterator_t>& p
// field is an array
struct token_index_meta_t {
- std::bitset<32> token_index;
+ std::bitset<128> token_index;
};
std::map<size_t, token_index_meta_t> array_index_to_token_index;
@@ -1540,7 +1540,7 @@ bool posting_list_t::has_prefix_match(std::vector<posting_list_t::iterator_t>& p
if (pos == prev_pos) { // indicates end of array index
size_t array_index = (size_t) offsets[start_offset_index];
- if (found_matching_index) {
+ if (found_matching_index && i+1 < 128) {
array_index_to_token_index[array_index].token_index.set(i + 1);
}
@@ -1630,7 +1630,7 @@ bool posting_list_t::has_exact_match(std::vector<posting_list_t::iterator_t>& po
// field is an array
struct token_index_meta_t {
- std::bitset<32> token_index;
+ std::bitset<128> token_index;
bool has_last_token;
};
@@ -1674,7 +1674,7 @@ bool posting_list_t::has_exact_match(std::vector<posting_list_t::iterator_t>& po
}
}
- if(found_matching_index) {
+ if(found_matching_index && i+1 < 128) {
array_index_to_token_index[array_index].token_index.set(i + 1);
}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index e704cec7..128f64c2 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -2927,6 +2927,36 @@ TEST_F(CollectionFilteringTest, PrefixFilterOnTextFields) {
ASSERT_EQ("Steve Reiley", res_obj["hits"][3]["document"]["names"][0]);
}
+TEST_F(CollectionFilteringTest, ExactFilterOnLongField) {
+ nlohmann::json schema = R"({
+ "name": "companies",
+ "fields": [
+ {"name": "keywords", "type": "string[]"}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+
+ auto coll = op.get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+
+ std::string arr_value;
+
+ // when value exceeds 128 tokens, we will fail gracefully
+ for(size_t i = 0; i < 130; i++) {
+ arr_value += "foo" + std::to_string(i) + " ";
+ }
+ doc1["keywords"] = {arr_value};
+
+ ASSERT_TRUE(coll->add(doc1.dump()).ok());
+
+ auto results = coll->search("*", {}, "keywords:=" + arr_value, {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(0, results["hits"].size());
+}
+
TEST_F(CollectionFilteringTest, FilterOnStemmedField) {
nlohmann::json schema = R"({
"name": "companies",