Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Clean up the match scoring logic.

Kishore Nallanc029e620d9b4
diff --git a/include/matchscore.h b/include/matchscore.h
index 02da896f..bb24cbdf 100644
--- a/include/matchscore.h
+++ b/include/matchscore.h
@@ -11,13 +11,13 @@
#define D(x)
#endif
-struct WordPosition {
- uint8_t word; // word identifier - one list for each word
- uint16_t position; // word's position in the text
- uint16_t index; // array index of the position in the list
+struct TokenPosition {
+ uint8_t token_id; // token identifier
+ uint16_t position; // token's position in the text
+ uint16_t position_index; // index of the position in the vector
- bool operator() (const WordPosition& a, const WordPosition& b) {
- return a.position > b.position;
+ bool operator() (const TokenPosition& a, const TokenPosition& b) {
+ return a.position > b.position;
}
};
@@ -26,80 +26,91 @@ struct MatchScore {
uint16_t distance;
};
-#define addTopOfHeapToWindow(heap,q,word_positions,word_pos_sum) {\
- WordPosition top = heap.top();\
+#define addTopOfHeapToWindow(heap,q,token_positions,token_pos) {\
+ TokenPosition top = heap.top();\
heap.pop();\
q.push(top);\
- word_pos_sum[top.word] = top.position; \
- top.index++;\
- /* push the next element from the same queue onto the heap (must check for bounds) */\
- if(top.index < word_positions[top.word].size()) {\
- heap.push(WordPosition{top.word, word_positions[top.word][top.index], top.index});\
+ token_pos[top.token_id] = top.position; \
+ top.position_index++;\
+ /* Must refill the heap - push the next position of the same token */\
+ if(top.position_index < token_positions[top.token_id].size()) {\
+ heap.push(TokenPosition{top.token_id, token_positions[top.token_id][top.position_index], top.position_index});\
}\
}
/*
- * Given *sorted positions* of target tokens in a document, generates a score that indicates:
+ * Given *sorted positions* of each target token in a *single* document, generates a score that indicates:
* a) How many tokens are present in the document
* b) The proximity between the tokens in the document
*
* We use a priority queue to read the position vectors in a sorted manner, slide a window of a given size, and
* compute the max_match and min_displacement of target tokens across the windows.
*/
-MatchScore match_score(std::vector<std::vector<uint16_t>> &word_positions) {
- std::priority_queue<WordPosition, std::vector<WordPosition>, WordPosition> heap;
-
- for(uint8_t word=0; word < word_positions.size(); word++) {
- heap.push(WordPosition{word, word_positions[word][0], 0});
+MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {
+ const size_t WINDOW_SIZE = 5;
+ const size_t MAX_TOKENS_IN_A_QUERY = 20;
+
+ std::priority_queue<TokenPosition, std::vector<TokenPosition>, TokenPosition> heap;
+
+ for(uint8_t token_id=0; token_id < token_positions.size(); token_id++) {
+ heap.push(TokenPosition{token_id, token_positions[token_id].front(), 0});
}
+ // heap now contains the first occurring position of each token in the given document
+
uint16_t max_match = 1;
uint16_t min_displacement = UINT16_MAX;
- std::queue<WordPosition> q;
- uint16_t word_pos_sum[20] = { };
+ std::queue<TokenPosition> q;
+ uint16_t token_pos[MAX_TOKENS_IN_A_QUERY] = { };
do {
if(q.empty()) {
- addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);
+ addTopOfHeapToWindow(heap, q, token_positions, token_pos);
}
D(cout << "Loop till window fills..." << endl;)
+ // Fill the queue with tokens within a given window frame size of the start position
+ // At the same time, we also record the *last* occurrence of each token within the window
+ // For e.g. if `cat` appeared at positions 1,3 and 5, we will record `word_pos_sum[cat] = 5`
const uint16_t start_pos = q.front().position;
- while(!heap.empty() && heap.top().position < start_pos+5) {
- addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);
+ while(!heap.empty() && heap.top().position < start_pos+WINDOW_SIZE) {
+ addTopOfHeapToWindow(heap, q, token_positions, token_pos);
}
+ D(cout << endl << "----" << endl);
+
uint16_t prev_pos = 0;
uint16_t num_match = 0;
uint16_t displacement = 0;
- size_t k = 0;
- D(cout << endl << "----" << endl);
-
- while(k < word_positions.size()) {
- if(word_pos_sum[k] != 0) {
- num_match++;
- if(prev_pos == 0) prev_pos = word_pos_sum[k];
- else {
- D(cout << "prev_pos: " << prev_pos << " , curr_pos: " << word_pos_sum[k] << endl);
- displacement += abs(word_pos_sum[k]-prev_pos);
- prev_pos = word_pos_sum[k];
- }
+ for(size_t token_id=0; token_id<token_positions.size(); token_id++) {
+ // If a token appeared within the window, we would have recorded its position
+ if(token_pos[token_id] != 0) {
+ num_match++;
+ if(prev_pos == 0) prev_pos = token_pos[token_id];
+ else {
+ // Calculate the distance between the tokens within the window
+ // Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other
+ D(cout << "prev_pos: " << prev_pos << " , curr_pos: " << token_pos[token_id] << endl);
+ displacement += abs(token_pos[token_id]-prev_pos);
+ prev_pos = token_pos[token_id];
}
- k++;
+ }
}
D(cout << endl << "!!!displacement: " << displacement << " | num_match: " << num_match << endl);
+ // Track the best `displacement` and `num_match` seen so far across all the windows
if(num_match >= max_match) {
- max_match = num_match;
- if(displacement != 0 && displacement < min_displacement) {
- min_displacement = displacement;
- }
+ max_match = num_match;
+ if(displacement != 0 && displacement < min_displacement) {
+ min_displacement = displacement;
+ }
}
- word_pos_sum[q.front().word] -= q.front().position;
+ // As we slide the window, drop the first token of the window from the computation
+ token_pos[q.front().token_id] = 0;
q.pop();
} while(!heap.empty());
diff --git a/src/main.cpp b/src/main.cpp
index ee73be77..5cde15c0 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1,6 +1,5 @@
#include <stdlib.h>
#include <iostream>
-#include <check.h>
#include <fstream>
#include <chrono>
#include <vector>
@@ -113,7 +112,6 @@ void find_documents(art_tree & t, string query, size_t max_results) {
}
}
- std::vector<std::vector<uint16_t>> word_positions;
Topster<100> topster;
size_t total_results = 0;
const size_t combination_limit = 10;
@@ -154,24 +152,26 @@ void find_documents(art_tree & t, string query, size_t max_results) {
// go through each matching document id and calculate match score
for(auto i=0; i<result_size; i++) {
- // we look up the doc_id in the token's doc index
- // and then arrive at the positions where the token occurs in every document
- for (art_leaf *token_leaf : query_suggestion) {
- vector<uint16_t> positions;
- uint32_t doc_index = token_leaf->values->ids.indexOf(result_ids[i]);
- uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);
- uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);
- for (auto offset_count = 1; offset_count <= num_offsets; offset_count++) {
- positions.push_back((uint16_t) token_leaf->values->offsets.at(offset_index + offset_count));
+ uint32_t doc_id = result_ids[i];
+ std::vector<std::vector<uint16_t>> token_positions;
+
+ // for each token in the query, find the positions that it appears in this document
+ for (art_leaf *token_leaf : query_suggestion) {
+ vector<uint16_t> positions;
+ uint32_t doc_index = token_leaf->values->ids.indexOf(doc_id);
+ uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);
+ uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);
+ for (auto offset_count = 1; offset_count <= num_offsets; offset_count++) {
+ positions.push_back((uint16_t) token_leaf->values->offsets.at(offset_index + offset_count));
+ }
+ token_positions.push_back(positions);
}
- word_positions.push_back(positions);
- }
- MatchScore score = match_score(word_positions);
- const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);
+ MatchScore score = match_score(token_positions);
+ const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);
- //cout << "result_ids[i]: " << result_ids[i] << " - cumulativeScore: " << cumulativeScore << endl;
- topster.add(result_ids[i], cumulativeScore);
+ cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << score.words_present << endl;
+ topster.add(doc_id, cumulativeScore);
}
total_results += result_size;