Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix memory corruption during unsorted append.

Kishore Nallan04d02919b255
diff --git a/include/forarray.h b/include/forarray.h
index 92627cb3..fcacf371 100644
--- a/include/forarray.h
+++ b/include/forarray.h
@@ -3,6 +3,7 @@
#include <stdio.h>
#include <cstdlib>
#include <for.h>
+#include <limits>
#define FOR_GROWTH_FACTOR 1.3
#define FOR_ELE_SIZE sizeof(uint32_t)
@@ -10,10 +11,12 @@
class forarray {
private:
+ uint8_t* in;
uint32_t size_bytes = 0;
uint32_t length_bytes = 0;
uint32_t length = 0;
- uint8_t* in;
+ uint32_t min = std::numeric_limits<uint32_t>::max();
+ uint32_t max = std::numeric_limits<uint32_t>::min();
public:
forarray(const uint32_t n=2) {
size_bytes = n * FOR_ELE_SIZE;
@@ -48,6 +51,10 @@ public:
length_bytes = new_length_bytes;
length++;
+
+ if(value < min) min = value;
+ if(value > max) max = value;
+
return true;
}
@@ -56,19 +63,8 @@ public:
return METADATA_OVERHEAD+FOR_ELE_SIZE;
}
- // calculate min/max
- uint32_t m = in[0];
- uint32_t M = m;
-
- for (uint32_t i = 1; i < length; i++) {
- if (in[i] < m)
- m = in[i];
- if (in[i] > M)
- M = in[i];
- }
-
- m = std::min(m, value);
- M = std::max(M, value);
+ uint32_t m = std::min(min, value);
+ uint32_t M = std::max(max, value);
uint32_t bnew = required_bits(M - m);
@@ -82,16 +78,26 @@ public:
// grow the array first
size_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);
uint8_t *new_location = (uint8_t *) realloc(in, new_size);
- if(new_location == NULL) return false;
+ if(new_location == NULL) {
+ abort();
+ return false;
+ }
in = new_location;
size_bytes = (uint32_t) new_size;
}
uint32_t new_length_bytes = for_append_unsorted(in, length, value);
- if(new_length_bytes == 0) return false;
+ if(new_length_bytes == 0) {
+ abort();
+ return false;
+ }
+
+ if (in[length] < min) min = in[length];
+ if (in[length] > max) max = in[length];
length_bytes = new_length_bytes;
length++;
+
return true;
}
diff --git a/include/matchscore.h b/include/matchscore.h
index bb24cbdf..50e68556 100644
--- a/include/matchscore.h
+++ b/include/matchscore.h
@@ -4,6 +4,7 @@
#include <vector>
#include <queue>
#include <stdlib.h>
+#include <limits>
#ifdef DEBUG
#define D(x) x
@@ -45,9 +46,10 @@ struct MatchScore {
* We use a priority queue to read the position vectors in a sorted manner, slide a window of a given size, and
* compute the max_match and min_displacement of target tokens across the windows.
*/
-MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {
- const size_t WINDOW_SIZE = 5;
+MatchScore match_score(uint32_t doc_id, std::vector<std::vector<uint16_t>> &token_positions) {
+ const size_t WINDOW_SIZE = 20;
const size_t MAX_TOKENS_IN_A_QUERY = 20;
+ const uint16_t MAX_UINT_16 = std::numeric_limits<uint16_t>::max();
std::priority_queue<TokenPosition, std::vector<TokenPosition>, TokenPosition> heap;
@@ -62,6 +64,7 @@ MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {
std::queue<TokenPosition> q;
uint16_t token_pos[MAX_TOKENS_IN_A_QUERY] = { };
+ std::fill_n(token_pos, MAX_TOKENS_IN_A_QUERY, MAX_UINT_16);
do {
if(q.empty()) {
@@ -72,7 +75,7 @@ MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {
// Fill the queue with tokens within a given window frame size of the start position
// At the same time, we also record the *last* occurrence of each token within the window
- // For e.g. if `cat` appeared at positions 1,3 and 5, we will record `word_pos_sum[cat] = 5`
+ // For e.g. if `cat` appeared at positions 1,3 and 5, we will record `token_pos[cat] = 5`
const uint16_t start_pos = q.front().position;
while(!heap.empty() && heap.top().position < start_pos+WINDOW_SIZE) {
addTopOfHeapToWindow(heap, q, token_positions, token_pos);
@@ -80,15 +83,15 @@ MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {
D(cout << endl << "----" << endl);
- uint16_t prev_pos = 0;
+ uint16_t prev_pos = MAX_UINT_16;
uint16_t num_match = 0;
uint16_t displacement = 0;
for(size_t token_id=0; token_id<token_positions.size(); token_id++) {
// If a token appeared within the window, we would have recorded its position
- if(token_pos[token_id] != 0) {
+ if(token_pos[token_id] != MAX_UINT_16) {
num_match++;
- if(prev_pos == 0) prev_pos = token_pos[token_id];
+ if(prev_pos == MAX_UINT_16) prev_pos = token_pos[token_id];
else {
// Calculate the distance between the tokens within the window
// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other
diff --git a/src/art.cpp b/src/art.cpp
index 90ca817c..c2d70e10 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -6,6 +6,7 @@
#include <assert.h>
#include <art.h>
#include <iostream>
+#include <limits>
#include <queue>
#include "art.h"
@@ -331,11 +332,12 @@ art_leaf* art_maximum(art_tree *t) {
static void add_document_to_leaf(const art_document *document, const art_leaf *leaf) {
leaf->values->ids.append_sorted(document->id);
uint32_t curr_index = leaf->values->offsets.getLength();
- leaf->values->offsets.append_sorted(document->offsets_len);
+ leaf->values->offset_index.append_sorted(curr_index);
+
+ leaf->values->offsets.append_unsorted(document->offsets_len);
for(uint32_t i=0; i<document->offsets_len; i++) {
leaf->values->offsets.append_unsorted(document->offsets[i]);
}
- leaf->values->offset_index.append_sorted(curr_index);
}
static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {
@@ -795,22 +797,31 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {
return NULL;
}
-static int topk_iter(art_node *root, int k, std::vector<art_leaf*> & results) {
+static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {
printf("INSIDE topk_iter: root->type: %d\n", root->type);
// FIXME: try to avoid branching
- auto cmp = [](art_node * left, art_node * right) {
+ auto cmp = [term_len](art_node * left, art_node * right) {
int lscore, rscore;
if (IS_LEAF(left)) {
art_leaf *l = (art_leaf *) LEAF_RAW(left);
- lscore = l->score;
+ // Exact matches should be boosted ahead of others
+ if(l->key_len == term_len) {
+ lscore = std::numeric_limits<uint16_t>::max();
+ } else {
+ lscore = l->score;
+ }
} else {
lscore = left->score;
}
if(IS_LEAF(right)) {
art_leaf *r = (art_leaf *) LEAF_RAW(right);
- rscore = r->score;
+ if(r->key_len == term_len) {
+ rscore = std::numeric_limits<uint16_t>::max();
+ } else {
+ rscore = r->score;
+ }
} else {
rscore = right->score;
}
@@ -851,14 +862,14 @@ static int topk_iter(art_node *root, int k, std::vector<art_leaf*> & results) {
break;
case NODE16:
- std::cout << "\nNODE16, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE16, SCORE: " << n->score << std::endl;
for (int i=0; i < n->num_children; i++) {
q.push(((art_node16*)n)->children[i]);
}
break;
case NODE48:
- std::cout << "\nNODE48, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE48, SCORE: " << n->score << std::endl;
for (int i=0; i < 256; i++) {
idx = ((art_node48*)n)->keys[i];
if (!idx) continue;
@@ -867,7 +878,7 @@ static int topk_iter(art_node *root, int k, std::vector<art_leaf*> & results) {
break;
case NODE256:
- std::cout << "\nNODE256, SCORE: " << n->score << std::endl;
+ //std::cout << "\nNODE256, SCORE: " << n->score << std::endl;
for (int i=0; i < 256; i++) {
if (!((art_node256*)n)->children[i]) continue;
q.push(((art_node256*)n)->children[i]);
@@ -1047,7 +1058,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call
/* reached end of term, and cost is below threshold, print children of this node as matches*/\
if(new_current_row[term_len] <= max_cost) {\
printf("START RECURSIVE ITER\n");\
- topk_iter(child, max_words, results);\
+ topk_iter(child, term_len, max_words, results);\
}\
} else if(row_min <= max_cost) {\
int new_depth = (child_char != 0) ? depth+1 : depth;\
@@ -1120,7 +1131,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
}
if(current_row[term_len] <= max_cost) {
- topk_iter(n, max_words, results);
+ topk_iter(n, term_len, max_words, results);
}
return 0;
@@ -1152,7 +1163,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,
if(depth == term_len) {
if(current_row[term_len] <= max_cost) {
printf("PARTIAL START RECURSIVE ITER\n");
- return topk_iter(n, max_words, results);
+ return topk_iter(n, term_len, max_words, results);
}
return 0;
diff --git a/src/main.cpp b/src/main.cpp
index 5cde15c0..0871dc73 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -8,9 +8,7 @@
#include <time.h>
#include <art.h>
#include <unordered_map>
-#include "art.h"
#include "topster.h"
-#include "forarray.h"
#include "intersection.h"
#include "matchscore.h"
#include "util.h"
@@ -83,6 +81,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_
document.offsets[i] = kv.second[i];
}
art_insert(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length(), &document);
+
delete document.offsets;
}
}
@@ -96,7 +95,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_
4. Intersect the lists to find docs that match each phrase
5. Sort the docs based on some ranking criteria
*/
-void find_documents(art_tree & t, string query, size_t max_results) {
+void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores, string query, size_t max_results) {
vector<string> tokens;
tokenize(query, tokens, " ", true);
@@ -167,10 +166,10 @@ void find_documents(art_tree & t, string query, size_t max_results) {
token_positions.push_back(positions);
}
- MatchScore score = match_score(token_positions);
- const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);
+ MatchScore mscore = match_score(doc_id, token_positions);
+ const uint32_t cumulativeScore = ((uint32_t)(mscore.words_present * 16 + (20 - mscore.distance)) * 64000) + docscores[doc_id];
- cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << score.words_present << endl;
+ //cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << mscore.words_present << endl;
topster.add(doc_id, cumulativeScore);
}
@@ -188,13 +187,17 @@ void find_documents(art_tree & t, string query, size_t max_results) {
uint32_t id = topster.getKeyAt(i);
cout << "ID: " << id << endl;
}
+
+ //cin.get();
}
int main() {
art_tree t;
art_tree_init(&t);
- std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");
+ unordered_map<uint32_t, uint16_t> docscores;
+
+ std::ifstream infile("/data/hnstories.tsv");
std::string line;
uint32_t doc_id = 1;
@@ -204,10 +207,16 @@ int main() {
tokenize(line, parts, "\t", true);
vector<string> tokens;
tokenize(parts[0], tokens, " ", true);
+
+ if(parts.size() != 2) continue;
+
+ docscores[doc_id] = (uint16_t) stoi(parts[1]);
index_document(t, doc_id, tokens, stoi(parts[1]));
doc_id++;
}
+ cout << "FINISHED INDEXING!" << endl << flush;
+
/*const unsigned char *prefix = (const unsigned char *) "the";
size_t prefix_len = strlen((const char *) prefix);
std::vector<art_leaf*> results;
@@ -229,7 +238,7 @@ int main() {
std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;
}*/
- find_documents(t, "are the", 10);
+ find_documents(t, docscores, "the apple", 10);
art_tree_destroy(&t);
return 0;