Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix memory corruption during unsorted append.
Kishore Nallan04d02919b255
diff --git a/include/forarray.h b/include/forarray.hindex 92627cb3..fcacf371 100644--- a/include/forarray.h+++ b/include/forarray.h@@ -3,6 +3,7 @@#include <stdio.h>#include <cstdlib>#include <for.h>+#include <limits>#define FOR_GROWTH_FACTOR 1.3#define FOR_ELE_SIZE sizeof(uint32_t)@@ -10,10 +11,12 @@class forarray {private:+ uint8_t* in;uint32_t size_bytes = 0;uint32_t length_bytes = 0;uint32_t length = 0;- uint8_t* in;+ uint32_t min = std::numeric_limits<uint32_t>::max();+ uint32_t max = std::numeric_limits<uint32_t>::min();public:forarray(const uint32_t n=2) {size_bytes = n * FOR_ELE_SIZE;@@ -48,6 +51,10 @@ public:length_bytes = new_length_bytes;length++;++ if(value < min) min = value;+ if(value > max) max = value;+return true;}@@ -56,19 +63,8 @@ public:return METADATA_OVERHEAD+FOR_ELE_SIZE;}- // calculate min/max- uint32_t m = in[0];- uint32_t M = m;-- for (uint32_t i = 1; i < length; i++) {- if (in[i] < m)- m = in[i];- if (in[i] > M)- M = in[i];- }-- m = std::min(m, value);- M = std::max(M, value);+ uint32_t m = std::min(min, value);+ uint32_t M = std::max(max, value);uint32_t bnew = required_bits(M - m);@@ -82,16 +78,26 @@ public:// grow the array firstsize_t new_size = (size_t) (size_required * FOR_GROWTH_FACTOR);uint8_t *new_location = (uint8_t *) realloc(in, new_size);- if(new_location == NULL) return false;+ if(new_location == NULL) {+ abort();+ return false;+ }in = new_location;size_bytes = (uint32_t) new_size;}uint32_t new_length_bytes = for_append_unsorted(in, length, value);- if(new_length_bytes == 0) return false;+ if(new_length_bytes == 0) {+ abort();+ return false;+ }++ if (in[length] < min) min = in[length];+ if (in[length] > max) max = in[length];length_bytes = new_length_bytes;length++;+return true;}diff --git a/include/matchscore.h b/include/matchscore.hindex bb24cbdf..50e68556 100644--- a/include/matchscore.h+++ b/include/matchscore.h@@ -4,6 +4,7 @@#include <vector>#include <queue>#include <stdlib.h>+#include <limits>#ifdef DEBUG#define D(x) x@@ -45,9 +46,10 @@ struct MatchScore {* We use a priority queue to read the position vectors in a sorted manner, slide a window of a given size, and* compute the max_match and min_displacement of target tokens across the windows.*/-MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {- const size_t WINDOW_SIZE = 5;+MatchScore match_score(uint32_t doc_id, std::vector<std::vector<uint16_t>> &token_positions) {+ const size_t WINDOW_SIZE = 20;const size_t MAX_TOKENS_IN_A_QUERY = 20;+ const uint16_t MAX_UINT_16 = std::numeric_limits<uint16_t>::max();std::priority_queue<TokenPosition, std::vector<TokenPosition>, TokenPosition> heap;@@ -62,6 +64,7 @@ MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {std::queue<TokenPosition> q;uint16_t token_pos[MAX_TOKENS_IN_A_QUERY] = { };+ std::fill_n(token_pos, MAX_TOKENS_IN_A_QUERY, MAX_UINT_16);do {if(q.empty()) {@@ -72,7 +75,7 @@ MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {// Fill the queue with tokens within a given window frame size of the start position// At the same time, we also record the *last* occurrence of each token within the window- // For e.g. if `cat` appeared at positions 1,3 and 5, we will record `word_pos_sum[cat] = 5`+ // For e.g. if `cat` appeared at positions 1,3 and 5, we will record `token_pos[cat] = 5`const uint16_t start_pos = q.front().position;while(!heap.empty() && heap.top().position < start_pos+WINDOW_SIZE) {addTopOfHeapToWindow(heap, q, token_positions, token_pos);@@ -80,15 +83,15 @@ MatchScore match_score(std::vector<std::vector<uint16_t>> &token_positions) {D(cout << endl << "----" << endl);- uint16_t prev_pos = 0;+ uint16_t prev_pos = MAX_UINT_16;uint16_t num_match = 0;uint16_t displacement = 0;for(size_t token_id=0; token_id<token_positions.size(); token_id++) {// If a token appeared within the window, we would have recorded its position- if(token_pos[token_id] != 0) {+ if(token_pos[token_id] != MAX_UINT_16) {num_match++;- if(prev_pos == 0) prev_pos = token_pos[token_id];+ if(prev_pos == MAX_UINT_16) prev_pos = token_pos[token_id];else {// Calculate the distance between the tokens within the window// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each otherdiff --git a/src/art.cpp b/src/art.cppindex 90ca817c..c2d70e10 100644--- a/src/art.cpp+++ b/src/art.cpp@@ -6,6 +6,7 @@#include <assert.h>#include <art.h>#include <iostream>+#include <limits>#include <queue>#include "art.h"@@ -331,11 +332,12 @@ art_leaf* art_maximum(art_tree *t) {static void add_document_to_leaf(const art_document *document, const art_leaf *leaf) {leaf->values->ids.append_sorted(document->id);uint32_t curr_index = leaf->values->offsets.getLength();- leaf->values->offsets.append_sorted(document->offsets_len);+ leaf->values->offset_index.append_sorted(curr_index);++ leaf->values->offsets.append_unsorted(document->offsets_len);for(uint32_t i=0; i<document->offsets_len; i++) {leaf->values->offsets.append_unsorted(document->offsets[i]);}- leaf->values->offset_index.append_sorted(curr_index);}static art_leaf* make_leaf(const unsigned char *key, uint32_t key_len, art_document *document) {@@ -795,22 +797,31 @@ void* art_delete(art_tree *t, const unsigned char *key, int key_len) {return NULL;}-static int topk_iter(art_node *root, int k, std::vector<art_leaf*> & results) {+static int topk_iter(art_node *root, int term_len, int k, std::vector<art_leaf*> & results) {printf("INSIDE topk_iter: root->type: %d\n", root->type);// FIXME: try to avoid branching- auto cmp = [](art_node * left, art_node * right) {+ auto cmp = [term_len](art_node * left, art_node * right) {int lscore, rscore;if (IS_LEAF(left)) {art_leaf *l = (art_leaf *) LEAF_RAW(left);- lscore = l->score;+ // Exact matches should be boosted ahead of others+ if(l->key_len == term_len) {+ lscore = std::numeric_limits<uint16_t>::max();+ } else {+ lscore = l->score;+ }} else {lscore = left->score;}if(IS_LEAF(right)) {art_leaf *r = (art_leaf *) LEAF_RAW(right);- rscore = r->score;+ if(r->key_len == term_len) {+ rscore = std::numeric_limits<uint16_t>::max();+ } else {+ rscore = r->score;+ }} else {rscore = right->score;}@@ -851,14 +862,14 @@ static int topk_iter(art_node *root, int k, std::vector<art_leaf*> & results) {break;case NODE16:- std::cout << "\nNODE16, SCORE: " << n->score << std::endl;+ //std::cout << "\nNODE16, SCORE: " << n->score << std::endl;for (int i=0; i < n->num_children; i++) {q.push(((art_node16*)n)->children[i]);}break;case NODE48:- std::cout << "\nNODE48, SCORE: " << n->score << std::endl;+ //std::cout << "\nNODE48, SCORE: " << n->score << std::endl;for (int i=0; i < 256; i++) {idx = ((art_node48*)n)->keys[i];if (!idx) continue;@@ -867,7 +878,7 @@ static int topk_iter(art_node *root, int k, std::vector<art_leaf*> & results) {break;case NODE256:- std::cout << "\nNODE256, SCORE: " << n->score << std::endl;+ //std::cout << "\nNODE256, SCORE: " << n->score << std::endl;for (int i=0; i < 256; i++) {if (!((art_node256*)n)->children[i]) continue;q.push(((art_node256*)n)->children[i]);@@ -1047,7 +1058,7 @@ int art_iter_prefix(art_tree *t, const unsigned char *key, int key_len, art_call/* reached end of term, and cost is below threshold, print children of this node as matches*/\if(new_current_row[term_len] <= max_cost) {\printf("START RECURSIVE ITER\n");\- topk_iter(child, max_words, results);\+ topk_iter(child, term_len, max_words, results);\}\} else if(row_min <= max_cost) {\int new_depth = (child_char != 0) ? depth+1 : depth;\@@ -1120,7 +1131,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,}if(current_row[term_len] <= max_cost) {- topk_iter(n, max_words, results);+ topk_iter(n, term_len, max_words, results);}return 0;@@ -1152,7 +1163,7 @@ static int art_iter_fuzzy_prefix_recurse(art_node *n, const unsigned char *term,if(depth == term_len) {if(current_row[term_len] <= max_cost) {printf("PARTIAL START RECURSIVE ITER\n");- return topk_iter(n, max_words, results);+ return topk_iter(n, term_len, max_words, results);}return 0;diff --git a/src/main.cpp b/src/main.cppindex 5cde15c0..0871dc73 100644--- a/src/main.cpp+++ b/src/main.cpp@@ -8,9 +8,7 @@#include <time.h>#include <art.h>#include <unordered_map>-#include "art.h"#include "topster.h"-#include "forarray.h"#include "intersection.h"#include "matchscore.h"#include "util.h"@@ -83,6 +81,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_document.offsets[i] = kv.second[i];}art_insert(&t, (const unsigned char *) kv.first.c_str(), (int) kv.first.length(), &document);+delete document.offsets;}}@@ -96,7 +95,7 @@ void index_document(art_tree& t, uint32_t doc_id, vector<string> tokens, uint16_4. Intersect the lists to find docs that match each phrase5. Sort the docs based on some ranking criteria*/-void find_documents(art_tree & t, string query, size_t max_results) {+void find_documents(art_tree & t, unordered_map<uint32_t, uint16_t>& docscores, string query, size_t max_results) {vector<string> tokens;tokenize(query, tokens, " ", true);@@ -167,10 +166,10 @@ void find_documents(art_tree & t, string query, size_t max_results) {token_positions.push_back(positions);}- MatchScore score = match_score(token_positions);- const uint16_t cumulativeScore = (const uint16_t) (score.words_present * 16 + score.distance);+ MatchScore mscore = match_score(doc_id, token_positions);+ const uint32_t cumulativeScore = ((uint32_t)(mscore.words_present * 16 + (20 - mscore.distance)) * 64000) + docscores[doc_id];- cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << score.words_present << endl;+ //cout << "result_ids[i]: " << result_ids[i] << " - score.words_present: " << mscore.words_present << endl;topster.add(doc_id, cumulativeScore);}@@ -188,13 +187,17 @@ void find_documents(art_tree & t, string query, size_t max_results) {uint32_t id = topster.getKeyAt(i);cout << "ID: " << id << endl;}++ //cin.get();}int main() {art_tree t;art_tree_init(&t);- std::ifstream infile("/Users/kishorenc/others/wreally/search/test/documents.txt");+ unordered_map<uint32_t, uint16_t> docscores;++ std::ifstream infile("/data/hnstories.tsv");std::string line;uint32_t doc_id = 1;@@ -204,10 +207,16 @@ int main() {tokenize(line, parts, "\t", true);vector<string> tokens;tokenize(parts[0], tokens, " ", true);++ if(parts.size() != 2) continue;++ docscores[doc_id] = (uint16_t) stoi(parts[1]);index_document(t, doc_id, tokens, stoi(parts[1]));doc_id++;}+ cout << "FINISHED INDEXING!" << endl << flush;+/*const unsigned char *prefix = (const unsigned char *) "the";size_t prefix_len = strlen((const char *) prefix);std::vector<art_leaf*> results;@@ -229,7 +238,7 @@ int main() {std::cout << ", Value: " << leaf->values->ids.at(0) << std::endl;}*/- find_documents(t, "are the", 10);+ find_documents(t, docscores, "the apple", 10);art_tree_destroy(&t);return 0;