Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Clean up the match scoring logic.
Kishore Nallanc029e620d9b4
diff --git a/include/matchscore.h b/include/matchscore.hindex 02da896f..bb24cbdf 100644--- a/include/matchscore.h+++ b/include/matchscore.h@@ -11,13 +11,13 @@#define D(x)#endif-struct WordPosition {- uint8_t word; // word identifier - one list for each word- uint16_t position; // word's position in the text- uint16_t index; // array index of the position in the list+struct TokenPosition {+ uint8_t token_id; // token identifier+ uint16_t position; // token's position in the text+ uint16_t position_index; // index of the position in the vector- bool operator() (const WordPosition& a, const WordPosition& b) {- return a.position > b.position;+ bool operator() (const TokenPosition& a, const TokenPosition& b) {+ return a.position > b.position;}};@@ -26,80 +26,91 @@ struct MatchScore {uint16_t distance;};-#define addTopOfHeapToWindow(heap,q,word_positions,word_pos_sum) {\- WordPosition top = heap.top();\+#define addTopOfHeapToWindow(heap,q,token_positions,token_pos) {\+ TokenPosition top = heap.top();\heap.pop();\q.push(top);\- word_pos_sum[top.word] = top.position; \- top.index++;\- /* push the next element from the same queue onto the heap (must check for bounds) */\- if(top.index < word_positions[top.word].size()) {\- heap.push(WordPosition{top.word, word_positions[top.word][top.index], top.index});\+ token_pos[top.token_id] = top.position; \+ top.position_index++;\+ /* Must refill the heap - push the next position of the same token */\+ if(top.position_index < token_positions[top.token_id].size()) {\+ heap.push(TokenPosition{top.token_id, token_positions[top.token_id][top.position_index], top.position_index});\}\}/*- * Given *sorted positions* of target tokens in a document, generates a score that indicates:+ * Given *sorted positions* of each target token in a *single* document, generates a score that indicates:* a) How many tokens are present in the document* b) The proximity between the tokens in the document** We use a priority queue to read the position vectors in a sorted manner, slide a window of a given size, and* compute the max_match and min_displacement of target tokens across the windows.*/-MatchScore match_score(std::vector<std::vector<uint16_t>> &word_positions) {- std::priority_queue<WordPosition, std::vector<WordPosition>, WordPosition> heap;-- for(uint8_t word=0; word < word_positions.size(); word++) {- heap.push(WordPosition{word, word_positions[word][0], 0});+MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {+ const size_t WINDOW_SIZE = 5;+ const size_t MAX_TOKENS_IN_A_QUERY = 20;++ std::priority_queue<TokenPosition, std::vector<TokenPosition>, TokenPosition> heap;++ for(uint8_t token_id=0; token_id < token_positions.size(); token_id++) {+ heap.push(TokenPosition{token_id, token_positions[token_id].front(), 0});}+ // heap now contains the first occurring position of each token in the given document+uint16_t max_match = 1;uint16_t min_displacement = UINT16_MAX;- std::queue<WordPosition> q;- uint16_t word_pos_sum[20] = { };+ std::queue<TokenPosition> q;+ uint16_t token_pos[MAX_TOKENS_IN_A_QUERY] = { };do {if(q.empty()) {- addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);+ addTopOfHeapToWindow(heap, q, token_positions, token_pos);}D(cout << "Loop till window fills..." << endl;)+ // Fill the queue with tokens within a given window frame size of the start position+ // At the same time, we also record the *last* occurrence of each token within the window+ // For e.g. if `cat` appeared at positions 1,3 and 5, we will record `word_pos_sum[cat] = 5`const uint16_t start_pos = q.front().position;- while(!heap.empty() && heap.top().position < start_pos+5) {- addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);+ while(!heap.empty() && heap.top().position < start_pos+WINDOW_SIZE) {+ addTopOfHeapToWindow(heap, q, token_positions, token_pos);}+ D(cout << endl << "----" << endl);+uint16_t prev_pos = 0;uint16_t num_match = 0;uint16_t displacement = 0;- size_t k = 0;- D(cout << endl << "----" << endl);-- while(k < word_positions.size()) {- if(word_pos_sum[k] != 0) {- num_match++;- if(prev_pos == 0) prev_pos = word_pos_sum[k];- else {- D(cout << "prev_pos: " << prev_pos << " , curr_pos: " << word_pos_sum[k] << endl);- displacement += abs(word_pos_sum[k]-prev_pos);- prev_pos = word_pos_sum[k];- }+ for(size_t token_id=0; token_id<token_positions.size(); token_id++) {+ // If a token appeared within the window, we would have recorded its position+ if(token_pos[token_id] != 0) {+ num_match++;+ if(prev_pos == 0) prev_pos = token_pos[token_id];+ else {+ // Calculate the distance between the tokens within the window+ // Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other+ D(cout << "prev_pos: " << prev_pos << " , curr_pos: " << token_pos[token_id] << endl);+ displacement += abs(token_pos[token_id]-prev_pos);+ prev_pos = token_pos[token_id];}- k++;+ }}D(cout << endl << "!!!displacement: " << displacement << " | num_match: " << num_match << endl);+ // Track the best `displacement` and `num_match` seen so far across all the windowsif(num_match >= max_match) {- max_match = num_match;- if(displacement != 0 && displacement < min_displacement) {- min_displacement = displacement;- }+ max_match = num_match;+ if(displacement != 0 && displacement < min_displacement) {+ min_displacement = displacement;+ }}- word_pos_sum[q.front().word] -= q.front().position;+ // As we slide the window, drop the first token of the window from the computation+ token_pos[q.front().token_id] = 0;q.pop();} while(!heap.empty());diff --git a/src/main.cpp b/src/main.cppindex ee73be77..5cde15c0 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -1,6 +1,5 @@#include <stdlib.h>#include <iostream>-#include <check.h>#include <fstream>#include <chrono>#include <vector>@@ -113,7 +112,6 @@ void find_documents(art_tree & t, string query, size_t max_results) {}}- std::vector<std::vector<uint16_t>> word_positions;Topster<100> topster;size_t total_results = 0;const size_t combination_limit = 10;@@ -154,24 +152,26 @@ void find_documents(art_tree & t, string query, size_t max_results) {// go through each matching document id and calculate match scorefor(auto i=0; i<result_size; i++) {- // we look up the doc_id in the token's doc index- // and then arrive at the positions where the token occurs in every document- for (art_leaf *token_leaf : query_suggestion) {- vector<uint16_t> positions;- uint32_t doc_index = token_leaf->values->ids.indexOf(result_ids[i]);- uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);- uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);- for (auto offset_count = 1; offset_count <= num_offsets; offset_count++) {- positions.push_back((uint16_t) token_leaf->values->offsets.at(offset_index + offset_count));+ uint32_t doc_id = result_ids[i];+ std::vector<std::vector<uint16_t>> token_positions;++ // for each token in the query, find the positions that it appears in this document+ for (art_leaf *token_leaf : query_suggestion) {+ vector<uint16_t> positions;+ uint32_t doc_index = token_leaf->values->ids.indexOf(doc_id);+ uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);+ uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);+ for (auto offset_count = 1; offset_count <= num_offsets; offset_count++) {+ positions.push_back((uint16_t) token_leaf->values->offsets.at(offset_index + offset_count));+ }+ token_positions.push_back(positions);}- word_positions.push_back(positions);- }- MatchScore score = match_score(word_positions);- const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);+ MatchScore score = match_score(token_positions);+ const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);- //cout << "result_ids[i]: " << result_ids[i] << " - cumulativeScore: " << cumulativeScore << endl;- topster.add(result_ids[i], cumulativeScore);+ cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << score.words_present << endl;+ topster.add(doc_id, cumulativeScore);}total_results += result_size;