Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix offset wrap around in match score.
Kishore Nallan71260ea2f0f2
diff --git a/cmake/TestResources.cmake b/cmake/TestResources.cmakeindex 6eac8048..5d424811 100644--- a/cmake/TestResources.cmake+++ b/cmake/TestResources.cmake@@ -4,6 +4,7 @@ set(ART_VERSION bbbf588bca55bce095538ee8ca8b422904baebc5)set(TEST_RESOURCES_DIR ${CMAKE_SOURCE_DIR}/build/test_resources)set(ART_WORDS_PATH ${TEST_RESOURCES_DIR}/words.txt)set(ART_UUID_PATH ${TEST_RESOURCES_DIR}/uuid.txt)+set(TOKEN_OFFSETS_PATH ${TEST_RESOURCES_DIR}/token_offsets.txt)if(NOT EXISTS ${TEST_RESOURCES_DIR})file(MAKE_DIRECTORY ${TEST_RESOURCES_DIR})@@ -18,3 +19,8 @@ if(NOT EXISTS ${ART_UUID_PATH} AND BUILD_DEPS STREQUAL "yes")message(STATUS "Downloading test resource - uuid.txt")file(DOWNLOAD https://raw.githubusercontent.com/kishorenc/libart/${ART_VERSION}/tests/uuid.txt ${ART_UUID_PATH})endif()++if(NOT EXISTS ${TOKEN_OFFSETS_PATH} AND BUILD_DEPS STREQUAL "yes")+ message(STATUS "Downloading test resource - token_offsets.txt")+ file(DOWNLOAD https://gist.githubusercontent.com/kishorenc/1d330714eb07019f210f16ccb3991217/raw/bd52e05375d305d5aaa7ac06219af999726933a4/token_offsets.log ${TOKEN_OFFSETS_PATH})+endif()diff --git a/include/match_score.h b/include/match_score.hindex 629ff2f0..5597fb64 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -19,7 +19,7 @@ struct token_positions_t {struct TokenOffset {uint8_t token_id; // token identifieruint16_t offset = MAX_DISPLACEMENT; // token's offset in the text- uint16_t offset_index; // index of the offset in the offset vector+ uint32_t offset_index; // index of the offset in the offset vectorbool operator()(const TokenOffset &a, const TokenOffset &b) {return a.offset > b.offset;@@ -162,7 +162,8 @@ struct Match {size_t this_num_match = 0;std::vector<TokenOffset> this_window(tokens_size);- uint8_t prev_token_id = window[0].token_id;+ uint16_t prev_offset = window[0].offset;+ bool all_offsets_are_same = true;for (size_t i = 0; i < window.size(); i++) {if(populate_window) {@@ -179,10 +180,12 @@ struct Match {this_window[window[i].token_id].offset = window[i].offset;}}++ all_offsets_are_same = all_offsets_are_same && (window[i].offset == prev_offset);}- if ( (this_num_match > best_num_match) ||- (this_num_match == best_num_match && this_displacement < best_displacement)) {+ if ( ((this_num_match > best_num_match) ||+ (this_num_match == best_num_match && this_displacement < best_displacement))) {best_displacement = this_displacement;best_num_match = this_num_match;max_offset = std::min((uint16_t)255, window.front().offset);@@ -209,7 +212,7 @@ struct Match {}// Push next offset of same token popped- uint16_t next_offset_index = (smallest_offset.offset_index + 1);+ uint32_t next_offset_index = (smallest_offset.offset_index + 1);TokenOffset token_offset{token_id, this_token_offsets[next_offset_index], next_offset_index};window.emplace_back(token_offset);}diff --git a/test/match_score_test.cpp b/test/match_score_test.cppindex 6306161a..364c856d 100644--- a/test/match_score_test.cpp+++ b/test/match_score_test.cpp@@ -2,6 +2,9 @@#include <gtest/gtest.h>#include <match_score.h>#include "posting_list.h"+#include <fstream>++#define token_offsets_file_path std::string(std::string(ROOT_DIR)+"/build/test_resources/token_offsets.txt").c_str()TEST(MatchTest, TokenOffsetsExceedWindowSize) {std::vector<token_positions_t> token_positions = {@@ -165,4 +168,26 @@ TEST(MatchTest, MatchScoreV2) {std::chrono::high_resolution_clock::now() - begin).count();LOG(INFO) << "Time taken: " << timeNanos;LOG(INFO) << total_distance << ", " << words_present << ", " << offset_sum;*/-}\ No newline at end of file+}++TEST(MatchTest, MatchScoreWithOffsetWrapAround) {+ std::vector<token_positions_t> token_offsets;++ std::ifstream infile(token_offsets_file_path);+ std::string line;++ while (std::getline(infile, line)) {+ if(line == "last_token:") {+ std::vector<uint16_t> positions;+ token_offsets.push_back(token_positions_t{false, positions});+ } else {+ token_offsets.back().positions.push_back(std::stoi(line));+ }+ }++ infile.close();++ auto match = Match(100, token_offsets, true, true);+ ASSERT_EQ(2, match.words_present);+ //ASSERT_EQ(2, match.distance);+}