Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Saving state.
Kishore Nallanf0f57f2e2d3d
diff --git a/include/forarray.h b/include/forarray.hindex 0da7df78..a2c5ac49 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -61,6 +61,11 @@ public:return actual == value;}+ uint32_t indexOf(uint32_t value) {+ uint32_t actual;+ return for_binary_search(in, length);+ }+uint32_t* uncompress() {uint32_t *out = new uint32_t[length];for_uncompress(in, out, length);diff --git a/include/matchscore.h b/include/matchscore.hnew file mode 100644index 00000000..2a08b267--- /dev/null+++ b/include/matchscore.h@@ -0,0 +1,107 @@+#pragma once++#include <stdint.h>+#include <vector>+#include <queue>+#include <stdlib.h>++#ifdef DEBUG+#define D(x) x+#else+#define D(x)+#endif++struct WordPosition {+ uint8_t word; // word identifier - one list for each word+ uint16_t position; // word's position in the text+ uint16_t index; // array index of the position in the list++ bool operator() (const WordPosition& a, const WordPosition& b) {+ return a.position > b.position;+ }+};++struct MatchScore {+ uint16_t words_present;+ uint16_t distance;+};++#define addTopOfHeapToWindow(heap,q,word_positions,word_pos_sum) {\+ WordPosition top = heap.top();\+ heap.pop();\+ q.push(top);\+ word_pos_sum[top.word] = top.position; \+ top.index++;\+ /* push the next element from the same queue onto the heap (must check for bounds) */\+ if(top.index < word_positions[top.word].size()) {\+ heap.push(WordPosition{top.word, word_positions[top.word][top.index], top.index});\+ }\+}+/*+ * Given *sorted positions* of target tokens in a document, generates a score that indicates:+ * a) How many tokens are present in the document+ * b) The proximity between the tokens in the document+ *+ * We use a priority queue to read the position vectors in a sorted manner, slide a window of a given size, and+ * compute the max_match and min_displacement of target tokens across the windows.+ */+MatchScore match_score(std::vector<std::vector<uint32_t>> &word_positions) {+ std::priority_queue<WordPosition, std::vector<WordPosition>, WordPosition> heap;++ for(uint8_t word=0; word < word_positions.size(); word++) {+ heap.push(WordPosition{word, word_positions[word][0], 0});+ }++ uint16_t max_match = 1;+ uint16_t min_displacement = UINT16_MAX;++ std::queue<WordPosition> q;+ uint16_t word_pos_sum[20] = { };++ do {+ if(q.empty()) {+ addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);+ }++ D(cout << "Loop till window fills..." << endl;)++ const uint16_t start_pos = q.front().position;+ while(!heap.empty() && heap.top().position < start_pos+5) {+ addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);+ }++ uint16_t prev_pos = 0;+ uint16_t num_match = 0;+ uint16_t displacement = 0;+ size_t k = 0;++ D(cout << endl << "----" << endl);++ while(k < word_positions.size()) {+ if(word_pos_sum[k] != 0) {+ num_match++;+ if(prev_pos == 0) prev_pos = word_pos_sum[k];+ else {+ D(cout << "prev_pos: " << prev_pos << " , curr_pos: " << word_pos_sum[k] << endl);+ displacement += abs(word_pos_sum[k]-prev_pos);+ prev_pos = word_pos_sum[k];+ }+ }+ k++;+ }++ D(cout << endl << "!!!displacement: " << displacement << " | num_match: " << num_match << endl);++ if(num_match >= max_match) {+ max_match = num_match;+ if(displacement != 0 && displacement < min_displacement) {+ min_displacement = displacement;+ }+ }++ word_pos_sum[q.front().word] -= q.front().position;+ q.pop();+ } while(!heap.empty());++ return MatchScore{max_match, min_displacement};+}diff --git a/src/main.cpp b/src/main.cppindex cab1f31a..ae9d5271 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -13,6 +13,7 @@#include "topster.h"#include "forarray.h"#include "intersection.h"+#include "matchscore.h"#include "util.h"using namespace std;@@ -99,7 +100,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint14. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-void find_documents(art_tree & t, string q) {+void find_documents(art_tree & t, string q, size_t max_results) {vector<string> tokens;tokenize(q, tokens, " ", true);@@ -117,6 +118,9 @@ void find_documents(art_tree & t, string q) {//cout << "token_leaves.size(): " << token_leaves.size() << endl;+ std::vector<std::vector<uint32_t>> word_positions;+ Topster<100> topster;+ size_t total_results = 0;const size_t combination_limit = 10;auto product = []( long long a, vector<art_leaf*>& b ) { return a*b.size(); };long long int N = accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product );@@ -141,6 +145,8 @@ void find_documents(art_tree & t, string q) {uint32_t* result = u[0]->values->ids.uncompress();size_t result_size = u[0]->values->ids.getLength();+ if(result_size == 0) continue;+for(auto i=1; i<u.size(); i++) {uint32_t* out = new uint32_t[result_size];uint32_t* curr = u[i]->values->ids.uncompress();@@ -150,9 +156,31 @@ void find_documents(art_tree & t, string q) {result = out;}+ // go through each document and calculate match score+ for(auto i=0; i<result_size; i++) {+ for (art_leaf *token_leaf : u) {+ vector<uint32_t> positions;+ uint32_t doc_index = token_leaf->values->ids.indexOf(result[i]);+ uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);+ uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);+ for (auto offset_count = 0; offset_count < num_offsets; offset_count++) {+ positions.push_back(token_leaf->values->offsets.at(offset_index + offset_count));+ }+ word_positions.push_back(positions);+ }++ MatchScore score = match_score(word_positions);+ topster.add((const uint32_t &) (score.words_present * 16 + score.distance));+ }++ total_results += result_size;cout << "RESULT SIZE: " << result_size << endl;delete result;++ if(total_results >= max_results) break;}++ //std::sort(topster.data);}int main() {@@ -194,7 +222,7 @@ int main() {//std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;}- find_documents(t, "are the");+ find_documents(t, "are the", 10);art_tree_destroy(&t);return 0;