Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor the way string array offsets are stored.

kishorenca73774b00560
diff --git a/include/index.h b/include/index.h
index 838cae79..6f962b66 100644
--- a/include/index.h
+++ b/include/index.h
@@ -249,8 +249,6 @@ public:
// strings under this length will be fully highlighted, instead of showing a snippet of relevant portion
enum {SNIPPET_STR_ABOVE_LEN = 30};
- enum {ARRAY_SEPARATOR = UINT16_MAX};
-
// Using a $ prefix so that these meta keys stay above record entries in a lexicographically ordered KV store
static constexpr const char* COLLECTION_META_PREFIX = "$CM";
static constexpr const char* COLLECTION_NEXT_SEQ_PREFIX = "$CS";
diff --git a/include/match_score.h b/include/match_score.h
index f9c1c601..8ff09068 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -94,8 +94,8 @@ struct Match {
/*
* Given *sorted offsets* of each target token in a *single* document, generates a score that indicates:
- * a) How many tokens are present in the document
- * b) The proximity between the tokens in the document
+ * a) How many tokens are present within a match window
+ * b) The proximity between the tokens within the match window
*
* We use a priority queue to read the offset vectors in a sorted manner, slide a window of a given size, and
* compute the max_match and min_displacement of target tokens across the windows.
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index e9bc3378..70d7dd22 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -118,6 +118,8 @@ Option<bool> CollectionManager::init(Store *store,
iter_batch.push_back(std::vector<index_record>());
}
+ int num_docs_read = 0;
+
while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
@@ -130,11 +132,13 @@ Option<bool> CollectionManager::init(Store *store,
return Option<bool>(false, "Bad JSON.");
}
+ num_docs_read++;
+
iter_batch[seq_id % collection->get_num_indices()].push_back(
index_record(0, seq_id, iter->value().ToString(), document)
);
- if(iter_batch.size() == init_batch_size) {
+ if(num_docs_read % init_batch_size == 0) {
batch_index_result res;
collection->par_index_in_memory(iter_batch, res);
for(size_t i = 0; i < collection->get_num_indices(); i++) {
diff --git a/src/index.cpp b/src/index.cpp
index 0ed910a4..6a5b5c42 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -450,36 +450,40 @@ void Index::index_string_field(const std::string & text, const uint32_t score, a
void Index::index_string_array_field(const std::vector<std::string> & strings, const uint32_t score, art_tree *t,
uint32_t seq_id, const bool verbatim) const {
- std::unordered_map<std::string, std::unordered_map<size_t, std::vector<uint32_t>>> token_array_positions;
+ std::unordered_map<std::string, std::vector<uint32_t>> token_positions;
for(size_t array_index = 0; array_index < strings.size(); array_index++) {
const std::string & str = strings[array_index];
std::vector<std::string> tokens;
std::string delim = verbatim ? "" : " ";
-
StringUtils::split(str, tokens, delim);
- for(uint32_t i=0; i<tokens.size(); i++) {
+ std::set<std::string> token_set; // required to deal with repeating tokens
+
+ // iterate and append offset positions
+ for(size_t i=0; i<tokens.size(); i++) {
auto & token = tokens[i];
+
if(!verbatim) {
string_utils.unicode_normalize(token);
}
- token_array_positions[token][array_index].push_back(i);
+
+ token_positions[token].push_back(i);
+ token_set.insert(token);
}
- }
- std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;
+ // repeat last element to indicate end of offsets for this array index
+ for(auto & token: token_set) {
+ token_positions[token].push_back(token_positions[token].back());
+ }
- for(const auto & kv: token_array_positions) {
- for(size_t array_index = 0; array_index < strings.size(); array_index++) {
- token_to_offsets[kv.first].insert(token_to_offsets[kv.first].end(),
- token_array_positions[kv.first][array_index].begin(),
- token_array_positions[kv.first][array_index].end());
- token_to_offsets[kv.first].push_back(ARRAY_SEPARATOR);
+ // iterate and append this array index to all tokens
+ for(auto & token: token_set) {
+ token_positions[token].push_back(array_index);
}
}
- insert_doc(score, t, seq_id, token_to_offsets);
+ insert_doc(score, t, seq_id, token_positions);
}
void Index::index_int32_array_field(const std::vector<int32_t> & values, const uint32_t score, art_tree *t,
@@ -1276,14 +1280,19 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_sugges
spp::sparse_hash_map<const art_leaf *, uint32_t *> &leaf_to_indices,
size_t result_index,
std::vector<std::vector<std::vector<uint16_t>>> &array_token_positions) {
+ if(query_suggestion.empty()) {
+ return ;
+ }
// array_token_positions:
// for every element in a potential array, for every token in query suggestion, get the positions
- // first let's ascertain the size of the array
- size_t array_size = 0;
+ // first ascertain the size of the array
+ size_t array_size = 1;
for (const art_leaf *token_leaf : query_suggestion) {
+ size_t this_array_size = 1;
+
uint32_t doc_index = leaf_to_indices.at(token_leaf)[result_index];
if(doc_index == token_leaf->values->ids.getLength()) {
continue;
@@ -1294,20 +1303,24 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_sugges
token_leaf->values->offsets.getLength() :
token_leaf->values->offset_index.at(doc_index+1);
- while(start_offset < end_offset) {
- uint16_t pos = (uint16_t) token_leaf->values->offsets.at(start_offset);
- if(pos == ARRAY_SEPARATOR) {
- array_size++;
- }
- start_offset++;
- }
+ if(end_offset - start_offset < 3) {
+ this_array_size = 1; // can only be a string since array needs atleast 3 positions for storage
+ } else {
+ // Array offset storage format:
+ // a) last element is array_index b) second and third last elements will be largest offset
+
+ auto last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 1);
+ auto second_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 2);
+ auto third_last_val = (uint16_t) token_leaf->values->offsets.at(end_offset - 3);
- if(array_size == 0) {
- // for plain string fields that don't use an ARRAY_SEPARATOR
- array_size = 1;
+ if(second_last_val != third_last_val) {
+ this_array_size = 1;
+ } else {
+ this_array_size = last_val + 1;
+ }
}
- break;
+ array_size = std::max(array_size, this_array_size);
}
// initialize array_token_positions
@@ -1325,28 +1338,32 @@ void Index::populate_token_positions(const std::vector<art_leaf *> &query_sugges
token_leaf->values->offsets.getLength() :
token_leaf->values->offset_index.at(doc_index+1);
- size_t array_index = 0;
std::vector<uint16_t> positions;
+ uint16_t prev_pos = -1;
while(start_offset < end_offset) {
- uint16_t pos = (uint16_t) token_leaf->values->offsets.at(start_offset);
+ auto pos = (uint16_t) token_leaf->values->offsets.at(start_offset);
start_offset++;
- if(pos == ARRAY_SEPARATOR) {
- if(positions.size() != 0) {
+ if(pos == prev_pos) { // indicates end of array index
+ if(!positions.empty()) {
+ size_t array_index = (uint16_t) token_leaf->values->offsets.at(start_offset);
array_token_positions[array_index].push_back(positions);
positions.clear();
}
- array_index++;
+
+ start_offset++; // skip current value which is array index
+ prev_pos = -1;
continue;
}
+ prev_pos = pos;
positions.push_back(pos);
}
- if(positions.size() != 0) {
- // for plain string fields that don't use an ARRAY_SEPARATOR
- array_token_positions[array_index].push_back(positions);
+ if(!positions.empty()) {
+ // for plain string fields
+ array_token_positions[0].push_back(positions);
}
}
}