Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Take care of token order in phrase search.

Kishore Nallan51a5f268e2f0
diff --git a/include/match_score.h b/include/match_score.h
index 21e4a209..19ef660f 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -38,6 +38,7 @@ struct Match {
uint8_t words_present;
uint8_t distance;
uint8_t exact_match;
+ uint8_t phrase_match;
std::vector<TokenOffset> offsets;
Match() : words_present(0), distance(0), exact_match(0) {
@@ -82,6 +83,13 @@ struct Match {
}
}
+ template<typename T>
+ void sort2(std::vector<T>& a) {
+ if(a[0] < a[1]) {
+ std::swap(a[0], a[1]);
+ }
+ }
+
template<typename T>
void sort3(std::vector<T>& a) {
if (a[0] > a[1]) {
@@ -144,10 +152,16 @@ struct Match {
size_t best_displacement = MAX_DISPLACEMENT;
while (window.size() > 1) {
- if(window.size() == 3) {
- sort3<TokenOffset>(window);
- } else {
- std::sort(window.begin(), window.end(), std::greater<TokenOffset>()); // descending comparator
+ switch(window.size()) {
+ case 2:
+ sort2<TokenOffset>(window);
+ break;
+ case 3:
+ sort3<TokenOffset>(window);
+ break;
+ default:
+ // descending comparator
+ std::sort(window.begin(), window.end(), std::greater<TokenOffset>());
}
size_t min_offset = window.back().offset;
@@ -156,6 +170,9 @@ struct Match {
size_t this_num_match = 0;
std::vector<TokenOffset> this_window(tokens_size);
+ uint8_t prev_token_id = window[0].token_id;
+ phrase_match = 1;
+
for (size_t i = 0; i < window.size(); i++) {
if(populate_window) {
this_window[window[i].token_id] = window[i];
@@ -171,6 +188,12 @@ struct Match {
this_window[window[i].token_id].offset = window[i].offset;
}
}
+
+ if(window[i].token_id > prev_token_id) {
+ phrase_match = 0;
+ }
+
+ prev_token_id = window[i].token_id;
}
if ( (this_num_match > best_num_match) ||
@@ -215,10 +238,13 @@ struct Match {
offsets = best_window;
}
+ if(best_num_match != tokens_size || distance != tokens_size-1) {
+ phrase_match = 0;
+ }
+
exact_match = 0;
if(check_exact_match) {
-
if(distance > token_offsets.size()-1) {
// we can exit early and don't have to care about other requirements
return;
diff --git a/src/index.cpp b/src/index.cpp
index 1750632f..3dd10e15 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3662,12 +3662,6 @@ inline uint32_t Index::next_suggestion(const std::vector<token_candidates> &toke
LOG(INFO) << ".";*/
}
- // Sort ascending based on matched documents for each token for faster intersection.
- // However, this causes the token order to deviate from original query's order.
- sort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {
- return posting_t::num_ids(left->values) < posting_t::num_ids(right->values);
- });
-
return total_cost;
}
diff --git a/src/posting_list.cpp b/src/posting_list.cpp
index bd2a324c..a8048a69 100644
--- a/src/posting_list.cpp
+++ b/src/posting_list.cpp
@@ -1247,7 +1247,9 @@ void posting_list_t::get_phrase_matches(std::vector<iterator_t>& its, bool field
for(auto& kv: array_token_positions) {
auto dist = Match(id, kv.second, false, false).distance;
- if(dist == its.size() - 1) {
+ bool is_phrase_match = dist == its.size() - 1;
+ is_phrase_match = Match(id, kv.second, false, false).phrase_match;
+ if(is_phrase_match) {
phrase_ids[phrase_id_index] = ids[i];
phrase_id_index++;
break;
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index c16d940e..f422514a 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -2178,6 +2178,19 @@ TEST_F(CollectionSpecificTest, PhraseSearch) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ // order of tokens in phrase must be respected
+ results = coll1->search(R"("train by the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search(R"("train the by")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search(R"("train the")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
+ results = coll1->search(R"("trooper state")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
+ ASSERT_EQ(0, results["hits"].size());
+
// two phrases
results = coll1->search(R"("by the" "then and")", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 10).get();
ASSERT_EQ(1, results["hits"].size());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 52658161..d92c1114 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -150,7 +150,7 @@ TEST_F(CollectionTest, ExactSearchShouldBeStable) {
ASSERT_EQ(0, results["found"].get<int>());
}
-TEST_F(CollectionTest, PhraseSearch) {
+TEST_F(CollectionTest, MultiTokenSearch) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("rocket launch", query_fields, "", facets, sort_fields, {0}, 10,
1, FREQUENCY,
@@ -257,7 +257,7 @@ TEST_F(CollectionTest, SearchWithExcludedTokens) {
ASSERT_EQ(20, results["found"].get<uint32_t>());
}
-TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
+TEST_F(CollectionTest, SkipUnindexedTokensDuringMultiTokenSearch) {
// Tokens that are not found in the index should be skipped
std::vector<std::string> facets;
nlohmann::json results = collection->search("DoesNotExist from", query_fields, "", facets, sort_fields, {0}, 10).get();
@@ -346,7 +346,7 @@ TEST_F(CollectionTest, SkipUnindexedTokensDuringPhraseSearch) {
ASSERT_EQ(0, results["hits"].size());
}
-TEST_F(CollectionTest, PartialPhraseSearch) {
+TEST_F(CollectionTest, PartialMultiTokenSearch) {
std::vector<std::string> facets;
nlohmann::json results = collection->search("rocket research", query_fields, "", facets,
sort_fields, {0}, 10, 1, FREQUENCY, {false}, 10).get();
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 0131f17e..34877d29 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -33,6 +33,7 @@ TEST(MatchTest, MatchScoreV2) {
auto match = Match(100, token_offsets, true);
ASSERT_EQ(4, match.words_present);
ASSERT_EQ(3, match.distance);
+ ASSERT_EQ(0, match.phrase_match);
std::vector<uint16_t> expected_offsets = {25, 26, 24, 27};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -54,6 +55,7 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(3, match.words_present);
ASSERT_EQ(2, match.distance);
ASSERT_EQ(0, match.exact_match);
+ ASSERT_EQ(0, match.phrase_match);
expected_offsets = {170, 171, 169};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -69,6 +71,7 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(1, match.words_present);
ASSERT_EQ(0, match.distance);
ASSERT_EQ(0, match.exact_match);
+ ASSERT_EQ(0, match.phrase_match);
expected_offsets = {38, MAX_DISPLACEMENT, MAX_DISPLACEMENT};
for(size_t i=0; i<token_offsets.size(); i++) {
@@ -92,11 +95,13 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(3, match.words_present);
ASSERT_EQ(2, match.distance);
ASSERT_EQ(1, match.exact_match);
+ ASSERT_EQ(0, match.phrase_match);
match = Match(100, token_offsets, true, false);
ASSERT_EQ(3, match.words_present);
ASSERT_EQ(2, match.distance);
ASSERT_EQ(0, match.exact_match);
+ ASSERT_EQ(0, match.phrase_match);
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {1}});
@@ -105,6 +110,7 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true, true);
ASSERT_EQ(0, match.exact_match);
+ ASSERT_EQ(1, match.phrase_match);
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {0}});
@@ -113,6 +119,7 @@ TEST(MatchTest, MatchScoreV2) {
match = Match(100, token_offsets, true, true);
ASSERT_EQ(0, match.exact_match);
+ ASSERT_EQ(1, match.phrase_match);
token_offsets.clear();
token_offsets.push_back(token_positions_t{false, {74}});
@@ -126,6 +133,23 @@ TEST(MatchTest, MatchScoreV2) {
ASSERT_EQ(expected_offsets[i], match.offsets[i].offset);
}
+ // check phrase match
+ token_offsets.clear();
+ token_offsets.push_back(token_positions_t{false, {38, 50, 187, 195, 201}});
+ token_offsets.push_back(token_positions_t{false, {120, 167, 171, 196}});
+ token_offsets.push_back(token_positions_t{true, {197, 250}});
+
+ match = Match(100, token_offsets);
+ ASSERT_EQ(1, match.phrase_match);
+
+ token_offsets.clear();
+ token_offsets.push_back(token_positions_t{false, {120, 167, 171, 196}});
+ token_offsets.push_back(token_positions_t{false, {38, 50, 187, 195, 201}});
+ token_offsets.push_back(token_positions_t{true, {197, 250}});
+
+ match = Match(100, token_offsets);
+ ASSERT_EQ(0, match.phrase_match);
+
/*size_t total_distance = 0, words_present = 0, offset_sum = 0;
auto begin = std::chrono::high_resolution_clock::now();