Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fixed a few bugs with calculation of best matched token offsets to be used for highlighting.

Kishore Nallan3efaed15b8da
diff --git a/include/match_score.h b/include/match_score.h
index 4c989420..d2c11c73 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -14,11 +14,6 @@
#define TokenOffsetHeap std::priority_queue<TokenOffset, std::vector<TokenOffset>, TokenOffset>
-union TokenOffsetDiffs {
- int16_t packed;
- char bytes[16];
-};
-
struct TokenOffset {
uint8_t token_id; // token identifier
uint16_t offset; // token's offset in the text
@@ -33,7 +28,12 @@ struct MatchScore {
uint16_t words_present;
uint16_t distance;
uint16_t start_offset;
- int16_t offset_diffs_packed;
+ char offset_diffs[16];
+
+ MatchScore(uint16_t words_present, uint16_t distance, uint16_t start_offset, char *offset_diffs_stacked):
+ words_present(words_present), distance(distance), start_offset(start_offset) {
+ memcpy(offset_diffs, offset_diffs_stacked, 16);
+ }
static void print_token_offsets(std::vector<std::vector<uint16_t>> &token_offsets) {
for(auto offsets: token_offsets) {
@@ -49,7 +49,7 @@ struct MatchScore {
TokenOffset top = heap.top();
heap.pop();
window.push(top);
- token_offset[top.token_id] = top.offset;
+ token_offset[top.token_id] = std::min(token_offset[top.token_id], top.offset);
top.offset_index++;
// Must refill the heap - push the next offset of the same token
@@ -59,10 +59,10 @@ struct MatchScore {
}
static void pack_token_offsets(const uint16_t* min_token_offset, const size_t num_tokens,
- TokenOffsetDiffs & offset_diffs) {
- offset_diffs.bytes[0] = num_tokens;
- for(size_t i = 1; i < num_tokens; i++) {
- offset_diffs.bytes[i] = (char)(min_token_offset[i] - min_token_offset[0]);
+ const size_t start_token_index, char *offset_diffs) {
+ offset_diffs[0] = (char) num_tokens;
+ for(size_t i = start_token_index; i < num_tokens; i++) {
+ offset_diffs[1+i] = (int8_t)(min_token_offset[i] - min_token_offset[start_token_index]);
}
}
@@ -86,7 +86,7 @@ struct MatchScore {
// heap now contains the first occurring offset of each token in the given document
- uint16_t max_match = 1;
+ uint16_t max_match = 0;
uint16_t min_displacement = MAX_DISPLACEMENT;
std::queue<TokenOffset> window;
@@ -95,6 +95,7 @@ struct MatchScore {
// used to store token offsets of the best-matched window
uint16_t min_token_offset[WINDOW_SIZE];
+ std::fill_n(min_token_offset, WINDOW_SIZE, MAX_DISPLACEMENT);
do {
if(window.empty()) {
@@ -123,6 +124,7 @@ struct MatchScore {
num_match++;
if(prev_pos == MAX_DISPLACEMENT) { // for the first word
prev_pos = token_offset[token_id];
+ displacement = 0;
} else {
// Calculate the distance between the tokens within the window
// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other
@@ -136,29 +138,36 @@ struct MatchScore {
D(std::cout << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match << std::endl);
// Track the best `displacement` and `num_match` seen so far across all the windows
- if(num_match >= max_match) {
+ // for a single token, displacement will be 0, while for 2 tokens minimum dispacement would be 1
+ if(num_match > max_match || (num_match == max_match && displacement < min_displacement)) {
+ min_displacement = displacement;
+ // record the token positions (for highlighting)
+ memcpy(min_token_offset, token_offset, token_offsets.size()*sizeof(uint16_t));
max_match = num_match;
- if(displacement == 0 || displacement < min_displacement) {
- // record the token positions (for highlighting)
- memcpy(min_token_offset, token_offset, token_offsets.size()*sizeof(uint16_t));
- }
-
- if(displacement != 0 && displacement < min_displacement) {
- min_displacement = displacement;
-
- }
}
// As we slide the window, drop the first token of the window from the computation
- token_offset[window.front().token_id] = 0;
+ token_offset[window.front().token_id] = MAX_DISPLACEMENT;
window.pop();
} while(!heap.empty());
// do run-length encoding of the min token positions/offsets
- TokenOffsetDiffs offset_diffs;
- uint16_t token_start_offset = min_token_offset[0];
- pack_token_offsets(min_token_offset, token_offsets.size(), offset_diffs);
+ uint16_t token_start_offset = 0;
+ char offset_diffs[16];
+ std::fill_n(offset_diffs, 16, 0);
+
+ int token_index = 0;
+
+ // identify the first token which is actually present and use that as the base for run-length encoding
+ while(token_index < token_offsets.size()) {
+ if(min_token_offset[token_index] != MAX_DISPLACEMENT) {
+ token_start_offset = min_token_offset[token_index];
+ break;
+ }
+ token_index++;
+ }
- return MatchScore{max_match, min_displacement, token_start_offset, offset_diffs.packed};
+ pack_token_offsets(min_token_offset, token_offsets.size(), token_index, offset_diffs);
+ return MatchScore(max_match, min_displacement, token_start_offset, offset_diffs);
}
};
diff --git a/include/topster.h b/include/topster.h
index cedcc114..8a5867b9 100644
--- a/include/topster.h
+++ b/include/topster.h
@@ -14,7 +14,7 @@ template <size_t MAX_SIZE=100>
struct Topster {
struct KV {
uint16_t start_offset;
- TokenOffsetDiffs offset_diffs; // [len, offset1-start_offset, offset2-start_offset, ...]
+ char offset_diffs[16]; // [len, offset1-start_offset, offset2-start_offset, ...]
uint64_t key;
uint64_t match_score;
int64_t primary_attr;
@@ -36,7 +36,7 @@ struct Topster {
}
void add(const uint64_t &key, const uint64_t &match_score, const int64_t &primary_attr,
- const int64_t &secondary_attr, const uint16_t &start_offset, const int16_t &offset_diffs_packed){
+ const int64_t &secondary_attr, const uint16_t &start_offset, char *offset_diffs_stacked){
if (size >= MAX_SIZE) {
if(!is_greater(data[0], match_score, primary_attr, secondary_attr)) {
// when incoming value is less than the smallest in the heap, ignore
@@ -56,7 +56,7 @@ struct Topster {
data[0].primary_attr = primary_attr;
data[0].secondary_attr = secondary_attr;
data[0].start_offset = start_offset;
- data[0].offset_diffs.packed = offset_diffs_packed;
+ memcpy(data[0].offset_diffs, offset_diffs_stacked, 16);
uint32_t i = 0;
// sift to maintain heap property
@@ -87,7 +87,7 @@ struct Topster {
data[size].primary_attr = primary_attr;
data[size].secondary_attr = secondary_attr;
data[size].start_offset = start_offset;
- data[size].offset_diffs.packed = offset_diffs_packed;
+ memcpy(data[size].offset_diffs, offset_diffs_stacked, 16);
size++;
for (uint32_t i = size - 1; i > 0;) {
diff --git a/src/collection.cpp b/src/collection.cpp
index d79706ba..8b34e67d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -660,11 +660,8 @@ nlohmann::json Collection::search(std::string query, const std::vector<std::stri
std::vector<std::string> tokens;
StringUtils::split(document[field_name], tokens, " ");
- tokens[field_order_kv.second.start_offset] =
- "<mark>" + tokens[field_order_kv.second.start_offset] + "</mark>";
-
- for(size_t i = 1; i < field_order_kv.second.offset_diffs.bytes[0]; i++) {
- size_t token_index = (size_t)(field_order_kv.second.start_offset + field_order_kv.second.offset_diffs.bytes[i]);
+ for(size_t i = 1; i <= field_order_kv.second.offset_diffs[0]; i++) {
+ size_t token_index = (size_t)(field_order_kv.second.start_offset + field_order_kv.second.offset_diffs[i]);
tokens[token_index] = "<mark>" + tokens[token_index] + "</mark>";
}
@@ -957,9 +954,10 @@ void Collection::score_results(const std::vector<sort_field> & sort_fields, cons
primary_rank_scores->at(seq_id) : 0;
int64_t secondary_rank_score = (secondary_rank_scores && secondary_rank_scores->count(seq_id) > 0) ?
secondary_rank_scores->at(seq_id) : 0;
+
topster.add(seq_id, match_score,
primary_rank_factor * primary_rank_score, secondary_rank_factor * secondary_rank_score,
- mscore.start_offset, mscore.offset_diffs_packed);
+ mscore.start_offset, mscore.offset_diffs);
/*std::cout << "candidate_rank_score: " << candidate_rank_score << ", words_present: " << mscore.words_present
<< ", match_score: " << match_score << ", primary_rank_score: " << primary_rank_score