Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor the way string array offsets are stored.
kishorenca73774b00560
diff --git a/include/index.h b/include/index.hindex 838cae79..6f962b66 100644--- a/include/index.h+++ b/include/index.h@@ -249,8 +249,6 @@ public:// strings under this length will be fully highlighted, instead of showing a snippet of relevant portionenum {SNIPPET_STR_ABOVE_LEN = 30};- enum {ARRAY_SEPARATOR = UINT16_MAX};-// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV storestatic constexpr const char* COLLECTION_META_PREFIX = "$CM";static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";diff --git a/include/match_score.h b/include/match_score.hindex f9c1c601..8ff09068 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -94,8 +94,8 @@ struct Match {/** Given *sorted offsets* of each target token in a *single* document, generates a score that indicates:- * a) How many tokens are present in the document- * b) The proximity between the tokens in the document+ * a) How many tokens are present within a match window+ * b) The proximity between the tokens within the match window** We use a priority queue to read the offset vectors in a sorted manner, slide a window of a given size, and* compute the max_match and min_displacement of target tokens across the windows.diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex e9bc3378..70d7dd22 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -118,6 +118,8 @@ Option<bool> CollectionManager::init(Store *store,iter_batch.push_back(std::vector<index_record>());}+ int num_docs_read = 0;+while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());@@ -130,11 +132,13 @@ Option<bool> CollectionManager::init(Store *store,return Option<bool>(false, "Bad JSON.");}+ num_docs_read++;+iter_batch[seq_id % collection->get_num_indices()].push_back(index_record(0, seq_id, iter->value().ToString(), document));- if(iter_batch.size() == init_batch_size) {+ if(num_docs_read % init_batch_size == 0) {batch_index_result res;collection->par_index_in_memory(iter_batch, res);for(size_t i = 0; i < collection->get_num_indices(); i++) {diff --git a/src/index.cpp b/src/index.cppindex 0ed910a4..6a5b5c42 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -450,36 +450,40 @@ void Index::index_string_field(const std::string & text, const uint32_t score, avoid Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,uint32_t seq_id, const bool verbatim) const {- std::unordered_map<std::string, std::unordered_map<size_t, std::vector<uint32_t>>> token_array_positions;+ std::unordered_map<std::string, std::vector<uint32_t>> token_positions;for(size_t array_index = 0; array_index < strings.size(); array_index++) {const std::string & str = strings[array_index];std::vector<std::string> tokens;std::string delim = verbatim ? "" : " ";-StringUtils::split(str, tokens, delim);- for(uint32_t i=0; i<tokens.size(); i++) {+ std::set<std::string> token_set; // required to deal with repeating tokens++ // iterate and append offset positions+ for(size_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];+if(!verbatim) {string_utils.unicode_normalize(token);}- token_array_positions[token][array_index].push_back(i);++ token_positions[token].push_back(i);+ token_set.insert(token);}- }- std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;+ // repeat last element to indicate end of offsets for this array index+ for(auto & token: token_set) {+ token_positions[token].push_back(token_positions[token].back());+ }- for(const auto & kv: token_array_positions) {- for(size_t array_index = 0; array_index < strings.size(); array_index++) {- token_to_offsets[kv.first].insert(token_to_offsets[kv.first].end(),- token_array_positions[kv.first][array_index].begin(),- token_array_positions[kv.first][array_index].end());- token_to_offsets[kv.first].push_back(ARRAY_SEPARATOR);+ // iterate and append this array index to all tokens+ for(auto & token: token_set) {+ token_positions[token].push_back(array_index);}}- insert_doc(score, t, seq_id, token_to_offsets);+ insert_doc(score, t, seq_id, token_positions);}void Index::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,@@ -1276,14 +1280,19 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggesspp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,size_t result_index,std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions) {+ if(query_suggestion.empty()) {+ return ;+ }// array_token_positions:// for every element in a potential array, for every token in query suggestion, get the positions- // first let's ascertain the size of the array- size_t array_size = 0;+ // first ascertain the size of the array+ size_t array_size = 1;for (const art_leaf *token_leaf : query_suggestion) {+ size_t this_array_size = 1;+uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];if(doc_index == token_leaf->values->ids.getLength()) {continue;@@ -1294,20 +1303,24 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggestoken_leaf->values->offsets.getLength() :token_leaf->values->offset_index.at(doc_index+1);- while(start_offset < end_offset) {- uint16_t pos = (uint16_t) token_leaf->values->offsets.at(start_offset);- if(pos == ARRAY_SEPARATOR) {- array_size++;- }- start_offset++;- }+ if(end_offset - start_offset < 3) {+ this_array_size = 1; // can only be a string since array needs atleast 3 positions for storage+ } else {+ // Array offset storage format:+ // a) last element is array_index b) second and third last elements will be largest offset++ auto last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 1);+ auto second_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 2);+ auto third_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 3);- if(array_size == 0) {- // for plain string fields that don't use an ARRAY_SEPARATOR- array_size = 1;+ if(second_last_val != third_last_val) {+ this_array_size = 1;+ } else {+ this_array_size = last_val + 1;+ }}- break;+ array_size = std::max(array_size, this_array_size);}// initialize array_token_positions@@ -1325,28 +1338,32 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_suggestoken_leaf->values->offsets.getLength() :token_leaf->values->offset_index.at(doc_index+1);- size_t array_index = 0;std::vector<uint16_t> positions;+ uint16_t prev_pos = -1;while(start_offset < end_offset) {- uint16_t pos = (uint16_t) token_leaf->values->offsets.at(start_offset);+ auto pos = (uint16_t) token_leaf->values->offsets.at(start_offset);start_offset++;- if(pos == ARRAY_SEPARATOR) {- if(positions.size() != 0) {+ if(pos == prev_pos) { // indicates end of array index+ if(!positions.empty()) {+ size_t array_index = (uint16_t) token_leaf->values->offsets.at(start_offset);array_token_positions[array_index].push_back(positions);positions.clear();}- array_index++;++ start_offset++; // skip current value which is array index+ prev_pos = -1;continue;}+ prev_pos = pos;positions.push_back(pos);}- if(positions.size() != 0) {- // for plain string fields that don't use an ARRAY_SEPARATOR- array_token_positions[array_index].push_back(positions);+ if(!positions.empty()) {+ // for plain string fields+ array_token_positions[0].push_back(positions);}}}