Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Saving state.

Kishore Nallanf0f57f2e2d3d
diff --git a/include/forarray.h b/include/forarray.h
index 0da7df78..a2c5ac49 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -61,6 +61,11 @@ public:
return actual == value;
}
+ uint32_t indexOf(uint32_t value) {
+ uint32_t actual;
+ return for_binary_search(in, length);
+ }
+
uint32_t* uncompress() {
uint32_t *out = new uint32_t[length];
for_uncompress(in, out, length);
diff --git a/include/matchscore.h b/include/matchscore.h
new file mode 100644
index 00000000..2a08b267
--- /dev/null
+++ b/include/matchscore.h
@@ -0,0 +1,107 @@
+#pragma once
+
+#include <stdint.h>
+#include <vector>
+#include <queue>
+#include <stdlib.h>
+
+#ifdef DEBUG
+#define D(x) x
+#else
+#define D(x)
+#endif
+
+struct WordPosition {
+ uint8_t word; // word identifier - one list for each word
+ uint16_t position; // word's position in the text
+ uint16_t index; // array index of the position in the list
+
+ bool operator() (const WordPosition& a, const WordPosition& b) {
+ return a.position > b.position;
+ }
+};
+
+struct MatchScore {
+ uint16_t words_present;
+ uint16_t distance;
+};
+
+#define addTopOfHeapToWindow(heap,q,word_positions,word_pos_sum) {\
+ WordPosition top = heap.top();\
+ heap.pop();\
+ q.push(top);\
+ word_pos_sum[top.word] = top.position; \
+ top.index++;\
+ /* push the next element from the same queue onto the heap (must check for bounds) */\
+ if(top.index < word_positions[top.word].size()) {\
+ heap.push(WordPosition{top.word, word_positions[top.word][top.index], top.index});\
+ }\
+}
+/*
+ * Given *sorted positions* of target tokens in a document, generates a score that indicates:
+ * a) How many tokens are present in the document
+ * b) The proximity between the tokens in the document
+ *
+ * We use a priority queue to read the position vectors in a sorted manner, slide a window of a given size, and
+ * compute the max_match and min_displacement of target tokens across the windows.
+ */
+MatchScore match_score(std::vector<std::vector<uint32_t>> &word_positions) {
+ std::priority_queue<WordPosition, std::vector<WordPosition>, WordPosition> heap;
+
+ for(uint8_t word=0; word < word_positions.size(); word++) {
+ heap.push(WordPosition{word, word_positions[word][0], 0});
+ }
+
+ uint16_t max_match = 1;
+ uint16_t min_displacement = UINT16_MAX;
+
+ std::queue<WordPosition> q;
+ uint16_t word_pos_sum[20] = { };
+
+ do {
+ if(q.empty()) {
+ addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);
+ }
+
+ D(cout << "Loop till window fills..." << endl;)
+
+ const uint16_t start_pos = q.front().position;
+ while(!heap.empty() && heap.top().position < start_pos+5) {
+ addTopOfHeapToWindow(heap, q, word_positions, word_pos_sum);
+ }
+
+ uint16_t prev_pos = 0;
+ uint16_t num_match = 0;
+ uint16_t displacement = 0;
+ size_t k = 0;
+
+ D(cout << endl << "----" << endl);
+
+ while(k < word_positions.size()) {
+ if(word_pos_sum[k] != 0) {
+ num_match++;
+ if(prev_pos == 0) prev_pos = word_pos_sum[k];
+ else {
+ D(cout << "prev_pos: " << prev_pos << " , curr_pos: " << word_pos_sum[k] << endl);
+ displacement += abs(word_pos_sum[k]-prev_pos);
+ prev_pos = word_pos_sum[k];
+ }
+ }
+ k++;
+ }
+
+ D(cout << endl << "!!!displacement: " << displacement << " | num_match: " << num_match << endl);
+
+ if(num_match >= max_match) {
+ max_match = num_match;
+ if(displacement != 0 && displacement < min_displacement) {
+ min_displacement = displacement;
+ }
+ }
+
+ word_pos_sum[q.front().word] -= q.front().position;
+ q.pop();
+ } while(!heap.empty());
+
+ return MatchScore{max_match, min_displacement};
+}
diff --git a/src/main.cpp b/src/main.cpp
index cab1f31a..ae9d5271 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -13,6 +13,7 @@
#include "topster.h"
#include "forarray.h"
#include "intersection.h"
+#include "matchscore.h"
#include "util.h"
using namespace std;
@@ -99,7 +100,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> & tokens, uint1
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void find_documents(art_tree & t, string q) {
+void find_documents(art_tree & t, string q, size_t max_results) {
vector<string> tokens;
tokenize(q, tokens, " ", true);
@@ -117,6 +118,9 @@ void find_documents(art_tree & t, string q) {
//cout << "token_leaves.size(): " << token_leaves.size() << endl;
+ std::vector<std::vector<uint32_t>> word_positions;
+ Topster<100> topster;
+ size_t total_results = 0;
const size_t combination_limit = 10;
auto product = []( long long a, vector<art_leaf*>& b ) { return a*b.size(); };
long long int N = accumulate(token_leaves.begin(), token_leaves.end(), 1LL, product );
@@ -141,6 +145,8 @@ void find_documents(art_tree & t, string q) {
uint32_t* result = u[0]->values->ids.uncompress();
size_t result_size = u[0]->values->ids.getLength();
+ if(result_size == 0) continue;
+
for(auto i=1; i<u.size(); i++) {
uint32_t* out = new uint32_t[result_size];
uint32_t* curr = u[i]->values->ids.uncompress();
@@ -150,9 +156,31 @@ void find_documents(art_tree & t, string q) {
result = out;
}
+ // go through each document and calculate match score
+ for(auto i=0; i<result_size; i++) {
+ for (art_leaf *token_leaf : u) {
+ vector<uint32_t> positions;
+ uint32_t doc_index = token_leaf->values->ids.indexOf(result[i]);
+ uint32_t offset_index = token_leaf->values->offset_index.at(doc_index);
+ uint32_t num_offsets = token_leaf->values->offsets.at(offset_index);
+ for (auto offset_count = 0; offset_count < num_offsets; offset_count++) {
+ positions.push_back(token_leaf->values->offsets.at(offset_index + offset_count));
+ }
+ word_positions.push_back(positions);
+ }
+
+ MatchScore score = match_score(word_positions);
+ topster.add((const uint32_t &) (score.words_present * 16 + score.distance));
+ }
+
+ total_results += result_size;
cout << "RESULT SIZE: " << result_size << endl;
delete result;
+
+ if(total_results >= max_results) break;
}
+
+ //std::sort(topster.data);
}
int main() {
@@ -194,7 +222,7 @@ int main() {
//std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;
}
- find_documents(t, "are the");
+ find_documents(t, "are the", 10);
art_tree_destroy(&t);
return 0;