Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fixed a few bugs with calculation of best matched token offsets to be used for highlighting.
Kishore Nallan3efaed15b8da
diff --git a/include/match_score.h b/include/match_score.hindex 4c989420..d2c11c73 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -14,11 +14,6 @@#define TokenOffsetHeap std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset>-union TokenOffsetDiffs {- int16_t packed;- char bytes[16];-};-struct TokenOffset {uint8_t token_id; // token identifieruint16_t offset; // token's offset in the text@@ -33,7 +28,12 @@ struct MatchScore {uint16_t words_present;uint16_t distance;uint16_t start_offset;- int16_t offset_diffs_packed;+ char offset_diffs[16];++ MatchScore(uint16_t words_present, uint16_t distance, uint16_t start_offset, char *offset_diffs_stacked):+ words_present(words_present), distance(distance), start_offset(start_offset) {+ memcpy(offset_diffs, offset_diffs_stacked, 16);+ }static void print_token_offsets(std::vector<std::vector<uint16_t>> &token_offsets) {for(auto offsets: token_offsets) {@@ -49,7 +49,7 @@ struct MatchScore {TokenOffset top = heap.top();heap.pop();window.push(top);- token_offset[top.token_id] = top.offset;+ token_offset[top.token_id] = std::min(token_offset[top.token_id], top.offset);top.offset_index++;// Must refill the heap - push the next offset of the same token@@ -59,10 +59,10 @@ struct MatchScore {}static void pack_token_offsets(const uint16_t* min_token_offset, const size_t num_tokens,- TokenOffsetDiffs & offset_diffs) {- offset_diffs.bytes[0] = num_tokens;- for(size_t i = 1; i < num_tokens; i++) {- offset_diffs.bytes[i] = (char)(min_token_offset[i] - min_token_offset[0]);+ const size_t start_token_index, char *offset_diffs) {+ offset_diffs[0] = (char) num_tokens;+ for(size_t i = start_token_index; i < num_tokens; i++) {+ offset_diffs[1+i] = (int8_t)(min_token_offset[i] - min_token_offset[start_token_index]);}}@@ -86,7 +86,7 @@ struct MatchScore {// heap now contains the first occurring offset of each token in the given document- uint16_t max_match = 1;+ uint16_t max_match = 0;uint16_t min_displacement = MAX_DISPLACEMENT;std::queue<TokenOffset> window;@@ -95,6 +95,7 @@ struct MatchScore {// used to store token offsets of the best-matched windowuint16_t min_token_offset[WINDOW_SIZE];+ std::fill_n(min_token_offset, WINDOW_SIZE, MAX_DISPLACEMENT);do {if(window.empty()) {@@ -123,6 +124,7 @@ struct MatchScore {num_match++;if(prev_pos == MAX_DISPLACEMENT) { // for the first wordprev_pos = token_offset[token_id];+ displacement = 0;} else {// Calculate the distance between the tokens within the window// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other@@ -136,29 +138,36 @@ struct MatchScore {D(std::cout << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match << std::endl);// Track the best `displacement` and `num_match` seen so far across all the windows- if(num_match >= max_match) {+ // for a single token, displacement will be 0, while for 2 tokens minimum dispacement would be 1+ if(num_match > max_match || (num_match == max_match && displacement < min_displacement)) {+ min_displacement = displacement;+ // record the token positions (for highlighting)+ memcpy(min_token_offset, token_offset, token_offsets.size()*sizeof(uint16_t));max_match = num_match;- if(displacement == 0 || displacement < min_displacement) {- // record the token positions (for highlighting)- memcpy(min_token_offset, token_offset, token_offsets.size()*sizeof(uint16_t));- }-- if(displacement != 0 && displacement < min_displacement) {- min_displacement = displacement;-- }}// As we slide the window, drop the first token of the window from the computation- token_offset[window.front().token_id] = 0;+ token_offset[window.front().token_id] = MAX_DISPLACEMENT;window.pop();} while(!heap.empty());// do run-length encoding of the min token positions/offsets- TokenOffsetDiffs offset_diffs;- uint16_t token_start_offset = min_token_offset[0];- pack_token_offsets(min_token_offset, token_offsets.size(), offset_diffs);+ uint16_t token_start_offset = 0;+ char offset_diffs[16];+ std::fill_n(offset_diffs, 16, 0);++ int token_index = 0;++ // identify the first token which is actually present and use that as the base for run-length encoding+ while(token_index < token_offsets.size()) {+ if(min_token_offset[token_index] != MAX_DISPLACEMENT) {+ token_start_offset = min_token_offset[token_index];+ break;+ }+ token_index++;+ }- return MatchScore{max_match, min_displacement, token_start_offset, offset_diffs.packed};+ pack_token_offsets(min_token_offset, token_offsets.size(), token_index, offset_diffs);+ return MatchScore(max_match, min_displacement, token_start_offset, offset_diffs);}};diff --git a/include/topster.h b/include/topster.hindex cedcc114..8a5867b9 100644--- a/include/topster.h+++ b/include/topster.h@@ -14,7 +14,7 @@ template <size_t MAX_SIZE=100>struct Topster {struct KV {uint16_t start_offset;- TokenOffsetDiffs offset_diffs; // [len, offset1-start_offset, offset2-start_offset, ...]+ char offset_diffs[16]; // [len, offset1-start_offset, offset2-start_offset, ...]uint64_t key;uint64_t match_score;int64_t primary_attr;@@ -36,7 +36,7 @@ struct Topster {}void add(const uint64_t &key, const uint64_t &match_score, const int64_t &primary_attr,- const int64_t &secondary_attr, const uint16_t &start_offset, const int16_t &offset_diffs_packed){+ const int64_t &secondary_attr, const uint16_t &start_offset, char *offset_diffs_stacked){if (size >= MAX_SIZE) {if(!is_greater(data[0], match_score, primary_attr, secondary_attr)) {// when incoming value is less than the smallest in the heap, ignore@@ -56,7 +56,7 @@ struct Topster {data[0].primary_attr = primary_attr;data[0].secondary_attr = secondary_attr;data[0].start_offset = start_offset;- data[0].offset_diffs.packed = offset_diffs_packed;+ memcpy(data[0].offset_diffs, offset_diffs_stacked, 16);uint32_t i = 0;// sift to maintain heap property@@ -87,7 +87,7 @@ struct Topster {data[size].primary_attr = primary_attr;data[size].secondary_attr = secondary_attr;data[size].start_offset = start_offset;- data[size].offset_diffs.packed = offset_diffs_packed;+ memcpy(data[size].offset_diffs, offset_diffs_stacked, 16);size++;for (uint32_t i = size - 1; i > 0;) {diff --git a/src/collection.cpp b/src/collection.cppindex d79706ba..8b34e67d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -660,11 +660,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stristd::vector<std::string> tokens;StringUtils::split(document[field_name], tokens, " ");- tokens[field_order_kv.second.start_offset] =- "<mark>" + tokens[field_order_kv.second.start_offset] + "</mark>";-- for(size_t i = 1; i < field_order_kv.second.offset_diffs.bytes[0]; i++) {- size_t token_index = (size_t)(field_order_kv.second.start_offset + field_order_kv.second.offset_diffs.bytes[i]);+ for(size_t i = 1; i <= field_order_kv.second.offset_diffs[0]; i++) {+ size_t token_index = (size_t)(field_order_kv.second.start_offset + field_order_kv.second.offset_diffs[i]);tokens[token_index] = "<mark>" + tokens[token_index] + "</mark>";}@@ -957,9 +954,10 @@ void Collection::score_results(const std::vector<sort_field> & sort_fields, consprimary_rank_scores->at(seq_id) : 0;int64_t secondary_rank_score = (secondary_rank_scores && secondary_rank_scores->count(seq_id) > 0) ?secondary_rank_scores->at(seq_id) : 0;+topster.add(seq_id, match_score,primary_rank_factor * primary_rank_score, secondary_rank_factor * secondary_rank_score,- mscore.start_offset, mscore.offset_diffs_packed);+ mscore.start_offset, mscore.offset_diffs);/*std::cout << "candidate_rank_score: " << candidate_rank_score << ", words_present: " << mscore.words_present<< ", match_score: " << match_score << ", primary_rank_score: " << primary_rank_score