Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix offset wrap around in match score.

Kishore Nallan71260ea2f0f2
diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmake
index 6eac8048..5d424811 100644
--- a/cmake/TestResources.cmake
+++ b/cmake/TestResources.cmake
@@ -4,6 +4,7 @@ set(ART_VERSION bbbf588bca55bce095538ee8ca8b422904baebc5)
set(TEST_RESOURCES_DIR ${CMAKE_SOURCE_DIR}/build/test_resources)
set(ART_WORDS_PATH ${TEST_RESOURCES_DIR}/words.txt)
set(ART_UUID_PATH ${TEST_RESOURCES_DIR}/uuid.txt)
+set(TOKEN_OFFSETS_PATH ${TEST_RESOURCES_DIR}/token_offsets.txt)
if(NOT EXISTS ${TEST_RESOURCES_DIR})
file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR})
@@ -18,3 +19,8 @@ if(NOT EXISTS ${ART_UUID_PATH} AND BUILD_DEPS STREQUAL "yes")
message(STATUS "Downloading test resource - uuid.txt")
file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/uuid.txt ${ART_UUID_PATH})
endif()
+
+if(NOT EXISTS ${TOKEN_OFFSETS_PATH} AND BUILD_DEPS STREQUAL "yes")
+ message(STATUS "Downloading test resource - token_offsets.txt")
+ file(DOWNLOAD https://gist.githubusercontent.com/kishorenc/1d330714eb07019f210f16ccb3991217/raw/bd52e05375d305d5aaa7ac06219af999726933a4/token_offsets.log ${TOKEN_OFFSETS_PATH})
+endif()
diff --git a/include/match_score.h b/include/match_score.h
index 629ff2f0..5597fb64 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -19,7 +19,7 @@ struct token_positions_t {
struct TokenOffset {
uint8_t token_id; // token identifier
uint16_t offset = MAX_DISPLACEMENT; // token's offset in the text
- uint16_t offset_index; // index of the offset in the offset vector
+ uint32_t offset_index; // index of the offset in the offset vector
bool operator()(const TokenOffset &a, const TokenOffset &b) {
return a.offset > b.offset;
@@ -162,7 +162,8 @@ struct Match {
size_t this_num_match = 0;
std::vector<TokenOffset> this_window(tokens_size);
- uint8_t prev_token_id = window[0].token_id;
+ uint16_t prev_offset = window[0].offset;
+ bool all_offsets_are_same = true;
for (size_t i = 0; i < window.size(); i++) {
if(populate_window) {
@@ -179,10 +180,12 @@ struct Match {
this_window[window[i].token_id].offset = window[i].offset;
}
}
+
+ all_offsets_are_same = all_offsets_are_same && (window[i].offset == prev_offset);
}
- if ( (this_num_match > best_num_match) ||
- (this_num_match == best_num_match && this_displacement < best_displacement)) {
+ if ( ((this_num_match > best_num_match) ||
+ (this_num_match == best_num_match && this_displacement < best_displacement))) {
best_displacement = this_displacement;
best_num_match = this_num_match;
max_offset = std::min((uint16_t)255, window.front().offset);
@@ -209,7 +212,7 @@ struct Match {
}
// Push next offset of same token popped
- uint16_t next_offset_index = (smallest_offset.offset_index + 1);
+ uint32_t next_offset_index = (smallest_offset.offset_index + 1);
TokenOffset token_offset{token_id, this_token_offsets[next_offset_index], next_offset_index};
window.emplace_back(token_offset);
}
diff --git a/test/match_score_test.cpp b/test/match_score_test.cpp
index 6306161a..364c856d 100644
--- a/test/match_score_test.cpp
+++ b/test/match_score_test.cpp
@@ -2,6 +2,9 @@
#include <gtest/gtest.h>
#include <match_score.h>
#include "posting_list.h"
+#include <fstream>
+
+#define token_offsets_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/token_offsets.txt").c_str()
TEST(MatchTest, TokenOffsetsExceedWindowSize) {
std::vector<token_positions_t> token_positions = {
@@ -165,4 +168,26 @@ TEST(MatchTest, MatchScoreV2) {
std::chrono::high_resolution_clock::now() - begin).count();
LOG(INFO) << "Time taken: " << timeNanos;
LOG(INFO) << total_distance << ", " << words_present << ", " << offset_sum;*/
-}
\ No newline at end of file
+}
+
+TEST(MatchTest, MatchScoreWithOffsetWrapAround) {
+ std::vector<token_positions_t> token_offsets;
+
+ std::ifstream infile(token_offsets_file_path);
+ std::string line;
+
+ while (std::getline(infile, line)) {
+ if(line == "last_token:") {
+ std::vector<uint16_t> positions;
+ token_offsets.push_back(token_positions_t{false, positions});
+ } else {
+ token_offsets.back().positions.push_back(std::stoi(line));
+ }
+ }
+
+ infile.close();
+
+ auto match = Match(100, token_offsets, true, true);
+ ASSERT_EQ(2, match.words_present);
+ //ASSERT_EQ(2, match.distance);
+}