Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix repeating tokens in phrase search.

Kishore Nallan10ad8162b678
diff --git a/include/match_score.h b/include/match_score.h
index 42e6bf45..629ff2f0 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -39,7 +39,6 @@ struct Match {
uint8_t distance = 0;
uint8_t max_offset = 0;
uint8_t exact_match = 0;
- uint8_t phrase_match = 0;
std::vector<TokenOffset> offsets;
@@ -49,7 +48,7 @@ struct Match {
Match(uint8_t words_present, uint8_t distance, uint8_t max_offset, uint8_t exact_match = 0) :
words_present(words_present), distance(distance), max_offset(max_offset),
- exact_match(exact_match), phrase_match(0) {
+ exact_match(exact_match) {
}
@@ -164,7 +163,6 @@ struct Match {
std::vector<TokenOffset> this_window(tokens_size);
uint8_t prev_token_id = window[0].token_id;
- phrase_match = 1;
for (size_t i = 0; i < window.size(); i++) {
if(populate_window) {
@@ -181,12 +179,6 @@ struct Match {
this_window[window[i].token_id].offset = window[i].offset;
}
}
-
- if(window[i].token_id > prev_token_id) {
- phrase_match = 0;
- }
-
- prev_token_id = window[i].token_id;
}
if ( (this_num_match > best_num_match) ||
@@ -232,10 +224,6 @@ struct Match {
offsets = best_window;
}
- if(best_num_match != tokens_size || distance != tokens_size-1) {
- phrase_match = 0;
- }
-
exact_match = 0;
if(check_exact_match) {
diff --git a/include/posting_list.h b/include/posting_list.h
index d044333e..76dc52c8 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -188,6 +188,11 @@ public:
const uint32_t* ids, const uint32_t num_ids,
uint32_t*& phrase_ids, size_t& num_phrase_ids);
+ static bool has_phrase_match(const std::vector<token_positions_t>& token_positions);
+
+ static bool found_token_sequence(const std::vector<token_positions_t>& token_positions, const size_t token_index,
+ const uint16_t target_pos);
+
static void get_matching_array_indices(uint32_t id, std::vector<iterator_t>& its,
std::vector<size_t>& indices);
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index 2de0fdde..8558375c 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1218,6 +1218,35 @@ void posting_list_t::get_exact_matches(std::vector<iterator_t>& its, const bool
num_exact_ids = exact_id_index;
}
+bool posting_list_t::found_token_sequence(const std::vector<token_positions_t>& token_positions,
+ const size_t token_index, const uint16_t target_pos) {
+
+ if(token_index == token_positions.size()) {
+ return true;
+ }
+
+ // iterate through the positions and see if `target_pos` is found in token positions
+ const auto& tok_positions = token_positions[token_index].positions;
+ auto pos_it = std::find(tok_positions.begin(), tok_positions.end(), target_pos);
+ if(pos_it == tok_positions.end()) {
+ return false;
+ }
+
+ return found_token_sequence(token_positions, token_index+1, target_pos+1);
+}
+
+bool posting_list_t::has_phrase_match(const std::vector<token_positions_t>& token_positions) {
+ const auto& positions = token_positions[0].positions;
+
+ for(auto pos: positions) {
+ if(found_token_sequence(token_positions, 1, pos + 1)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field_is_array, const uint32_t* ids,
const uint32_t num_ids, uint32_t*& phrase_ids, size_t& num_phrase_ids) {
@@ -1241,8 +1270,8 @@ void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field
get_offsets(its, array_token_positions);
for(auto& kv: array_token_positions) {
- bool is_phrase_match = Match(id, kv.second, false, false).phrase_match;
- if(is_phrase_match) {
+ const auto& token_positions = kv.second;
+ if(has_phrase_match(token_positions)) {
phrase_ids[phrase_id_index] = ids[i];
phrase_id_index++;
break;
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 2c82f7ed..4cfed868 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -1543,3 +1543,51 @@ TEST_F(CollectionSpecificMoreTest, IncludeExcludeUnIndexedField) {
ASSERT_EQ("Sample Title 1", res["hits"][0]["document"]["title"].get<std::string>());
ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
}
+
+TEST_F(CollectionSpecificMoreTest, PhraseMatchRepeatingTokens) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["title"] = "Super easy super fast product";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["title"] = "The really easy really fast product really";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto res = coll1->search(R"("super easy super fast")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+
+ res = coll1->search(R"("super easy super")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+
+ res = coll1->search(R"("the really easy really fast product really")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("1", res["hits"][0]["document"]["id"].get<std::string>());
+
+ // these should not match
+ res = coll1->search(R"("the easy really really product fast really")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ res = coll1->search(R"("really the easy really fast product really")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ res = coll1->search(R"("super super easy fast")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ res = coll1->search(R"("super super easy")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(0, res["hits"].size());
+
+ res = coll1->search(R"("product fast")", {"title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(0, res["hits"].size());
+}
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 34877d29..6306161a 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -1,6 +1,7 @@
#include <chrono>
#include <gtest/gtest.h>
#include <match_score.h>
+#include "posting_list.h"
TEST(MatchTest, TokenOffsetsExceedWindowSize) {
std::vector<token_positions_t> token_positions = {
@@ -33,7 +34,7 @@ TEST(MatchTest, MatchScoreV2) {
auto match = Match(100, token_offsets, true);
ASSERT_EQ(4, match.words_present);
ASSERT_EQ(3, match.distance);
- ASSERT_EQ(0, match.phrase_match);
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -55,7 +56,7 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(3, match.words_present);
ASSERT_EQ(2, match.distance);
ASSERT_EQ(0, match.exact_match);
- ASSERT_EQ(0, match.phrase_match);
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
expected_offsets = {170, 171, 169};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -71,7 +72,7 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(1, match.words_present);
ASSERT_EQ(0, match.distance);
ASSERT_EQ(0, match.exact_match);
- ASSERT_EQ(0, match.phrase_match);
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -95,13 +96,13 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(3, match.words_present);
ASSERT_EQ(2, match.distance);
ASSERT_EQ(1, match.exact_match);
- ASSERT_EQ(0, match.phrase_match);
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
match = Match(100, token_offsets, true, false);
ASSERT_EQ(3, match.words_present);
ASSERT_EQ(2, match.distance);
ASSERT_EQ(0, match.exact_match);
- ASSERT_EQ(0, match.phrase_match);
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {1}});
@@ -110,7 +111,7 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true, true);
ASSERT_EQ(0, match.exact_match);
- ASSERT_EQ(1, match.phrase_match);
+ ASSERT_TRUE(posting_list_t::has_phrase_match(token_offsets));
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {0}});
@@ -119,7 +120,7 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true, true);
ASSERT_EQ(0, match.exact_match);
- ASSERT_EQ(1, match.phrase_match);
+ ASSERT_TRUE(posting_list_t::has_phrase_match(token_offsets));
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {74}});
@@ -140,7 +141,7 @@ TEST(MatchTest, MatchScoreV2) {
token_offsets.push_back(token_positions_t{true, {197, 250}});
match = Match(100, token_offsets);
- ASSERT_EQ(1, match.phrase_match);
+ ASSERT_TRUE(posting_list_t::has_phrase_match(token_offsets));
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {120, 167, 171, 196}});
@@ -148,7 +149,7 @@ TEST(MatchTest, MatchScoreV2) {
token_offsets.push_back(token_positions_t{true, {197, 250}});
match = Match(100, token_offsets);
- ASSERT_EQ(0, match.phrase_match);
+ ASSERT_FALSE(posting_list_t::has_phrase_match(token_offsets));
/*size_t total_distance = 0, words_present = 0, offset_sum = 0;
auto begin = std::chrono::high_resolution_clock::now();