Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

optimization with new index

krunal13134127c80aea51
diff --git a/include/facet_index.h b/include/facet_index.h
index c97d6097..124eaf72 100644
--- a/include/facet_index.h
+++ b/include/facet_index.h
@@ -43,8 +43,7 @@ private:
struct facet_index_counter {
tsl::htrie_map<char, facet_index_struct> facet_index_map;
- std::vector<count_list> counter_list;
- bool is_migrated = false;
+ std::vector<count_list*> counter_list;
facet_index_counter() {
facet_index_map.clear();
@@ -57,7 +56,10 @@ private:
}
facet_index_map.clear();
-
+
+ for(auto val : counter_list) {
+ delete val;
+ }
counter_list.clear();
}
};
@@ -70,7 +72,8 @@ public:
~facet_index_t();
- uint32_t insert(const std::string& field, const std::string& value, uint32_t id, bool is_string = false);
+ void insert(const std::string& field, const std::string& value,
+ const std::vector<uint32_t>& ids, uint32_t index);
void erase(const std::string& field);
diff --git a/include/index.h b/include/index.h
index d2f725fc..4630cd63 100644
--- a/include/index.h
+++ b/include/index.h
@@ -205,6 +205,8 @@ struct index_record {
std::unordered_map<std::string, std::vector<uint32_t>>> field_index;
int64_t points;
+ std::map<std::string, std::vector<uint32_t>> facet_hashes;
+
Option<bool> indexed; // indicates if the indexing operation was a success
DIRTY_VALUES dirty_values;
@@ -308,6 +310,10 @@ private:
facet_index_t* facet_index_v4 = nullptr;
+ //for string and int64 facets insertions
+ uint32_t count_index;
+
+ std::unordered_map<std::string, uint32_t> count_index_map;
// sort_field => (seq_id => value)
spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;
@@ -608,7 +614,8 @@ public:
static void compute_token_offsets_facets(index_record& record,
const tsl::htrie_map<char, field>& search_schema,
const std::vector<char>& local_token_separators,
- const std::vector<char>& local_symbols_to_index);
+ const std::vector<char>& local_symbols_to_index,
+ std::map<std::string, std::vector<uint32_t>>& facet_hashes);
static void scrub_reindex_doc(const tsl::htrie_map<char, field>& search_schema,
nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);
@@ -674,7 +681,8 @@ public:
const std::vector<char>& symbols_to_index,
const bool do_validation);
- void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);
+ void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch,
+ std::map<std::string, std::vector<uint32_t>>& facet_hashes);
template<class T>
void iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func);
diff --git a/src/facet_index.cpp b/src/facet_index.cpp
index 88e8ec69..2719a134 100644
--- a/src/facet_index.cpp
+++ b/src/facet_index.cpp
@@ -10,75 +10,63 @@ void facet_index_t::initialize(const std::string& field) {
}
}
-uint32_t facet_index_t::insert(const std::string& field, const std::string& value,
- uint32_t id, bool is_string) {
+void facet_index_t::insert(const std::string& field, const std::string& value,
+ const std::vector<uint32_t>& ids, uint32_t index) {
const auto facet_field_map_it = facet_field_map.find(field);
if(facet_field_map_it == facet_field_map.end()) {
- return 0; //field is not initialized or dropped
+ return; //field is not initialized or dropped
}
- uint32_t index = 0;
-
auto& facet_index_map = facet_field_map_it->second.facet_index_map;
const auto sv = value.substr(0, 100);
const auto it = facet_index_map.find(sv);
if(it == facet_index_map.end()) {
- index = ++count_index;
facet_index_struct fis{};
fis.index = index;
- if(is_string) {
- fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));
- }
+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(ids.size(), ids));
facet_index_map.emplace(sv, fis);
} else {
- if(is_string) {
- auto ids = it->id_list_ptr;
- if (!ids_t::contains(ids, id)) {
- ids_t::upsert(ids, id);
- facet_index_map[sv].id_list_ptr = ids;
+ auto ids_ptr = it->id_list_ptr;
+ for(const auto& id : ids) {
+ if (!ids_t::contains(ids_ptr, id)) {
+ ids_t::upsert(ids_ptr, id);
+ facet_index_map[sv].id_list_ptr = ids_ptr;
}
}
- index = facet_index_map[sv].index;
}
- if(is_string) {
- const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
- //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
- auto& counter_list = facet_field_map_it->second.counter_list;
+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);
+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;
+ auto& counter_list = facet_field_map_it->second.counter_list;
- if(counter_list.empty()) {
- counter_list.emplace_back(sv, facet_count, index);
- } else {
- count_list node(sv, facet_count, index);
- auto ind = 0;
-
- for(; ind < counter_list.size(); ++ind) {
- if(counter_list[ind].index == index) {
- counter_list[ind].count = facet_count;
-
- if(ind > 1) {
- auto curr = ind;
- while (curr && (counter_list[curr-1].count < counter_list[curr].count)) {
- count_list temp = counter_list[curr-1];
- counter_list[curr-1] = counter_list[curr];
- counter_list[curr] = temp;
- --curr;
- }
+ if(counter_list.empty()) {
+ count_list* node = new count_list(sv, facet_count, index);
+ counter_list.emplace_back(node);
+ } else {
+ auto ind = 0;
+
+ for(; ind < counter_list.size(); ++ind) {
+ if(counter_list[ind]->index == index) {
+ counter_list[ind]->count = facet_count;
+ if(ind > 1) {
+ auto curr = ind;
+ while (curr && (counter_list[curr-1]->count < counter_list[curr]->count)) {
+ std::swap(counter_list[curr-1], counter_list[curr]);
+ --curr;
}
- break;
}
- }
- if(ind == counter_list.size()) {
- // LOG (INFO) << "inserting at last facet " << node.facet_value
- // << " with count " << node.count;
- counter_list.emplace_back(node);
+ break;
}
}
+ if(ind == counter_list.size()) {
+ // LOG (INFO) << "inserting at last facet " << node.facet_value
+ // << " with count " << node.count;
+ count_list* node = new count_list(sv, facet_count, index);
+ counter_list.emplace_back(node);
+ }
}
-
- return index;
}
bool facet_index_t::contains(const std::string& field) {
@@ -137,7 +125,7 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* result
}
const auto facet_index_map = facet_field_it->second.facet_index_map;
- const auto counter_list = facet_field_it->second.counter_list;
+ const auto& counter_list = facet_field_it->second.counter_list;
// LOG (INFO) << "facet_index_map size " << facet_index_map.size()
// << " , counter_list size " << counter_list.size();
@@ -150,9 +138,9 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* result
uint32_t count = 0;
if(is_wildcard_no_filter_query) {
- count = counter_list_it.count;
+ count = counter_list_it->count;
} else {
- auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;
+ auto ids = facet_index_map.at(counter_list_it->facet_value).id_list_ptr;
ids_t::uncompress(ids, id_list);
const auto ids_len = id_list.size();
for(int i = 0; i < result_ids_len; ++i) {
@@ -165,7 +153,7 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* result
}
if(count) {
- found[counter_list_it.facet_value] = count;
+ found[counter_list_it->facet_value] = count;
if(found.size() == max_facets) {
break;
}
diff --git a/src/index.cpp b/src/index.cpp
index 881d7a64..a6cee7c3 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -246,7 +246,8 @@ float Index::int64_t_to_float(int64_t n) {
void Index::compute_token_offsets_facets(index_record& record,
const tsl::htrie_map<char, field>& search_schema,
const std::vector<char>& local_token_separators,
- const std::vector<char>& local_symbols_to_index) {
+ const std::vector<char>& local_symbols_to_index,
+ std::map<std::string, std::vector<uint32_t>>& facet_hashes) {
const auto& document = record.doc;
@@ -267,19 +268,27 @@ void Index::compute_token_offsets_facets(index_record& record,
if(the_field.type == field_types::INT32_ARRAY) {
for(int32_t value: document[field_name]){
- strings.push_back(std::to_string(value));
+ auto str = std::to_string(value);
+ facet_hashes[str].emplace_back(record.seq_id);
+ strings.emplace_back(std::move(str));
}
} else if(the_field.type == field_types::INT64_ARRAY) {
for(int64_t value: document[field_name]){
- strings.push_back(std::to_string(value));
+ auto str = std::to_string(value);
+ facet_hashes[str].emplace_back(record.seq_id);
+ strings.emplace_back(std::move(str));
}
} else if(the_field.type == field_types::FLOAT_ARRAY) {
for(float value: document[field_name]){
- strings.push_back(StringUtils::float_to_str(value));
+ auto str = StringUtils::float_to_str(value);
+ facet_hashes[str].emplace_back(record.seq_id);
+ strings.emplace_back(std::move(str));
}
} else if(the_field.type == field_types::BOOL_ARRAY) {
for(bool value: document[field_name]){
- strings.push_back(std::to_string(value));
+ auto str = std::to_string(value);
+ facet_hashes[str].emplace_back(record.seq_id);
+ strings.emplace_back(std::move(str));
}
}
@@ -290,15 +299,21 @@ void Index::compute_token_offsets_facets(index_record& record,
std::string text;
if(the_field.type == field_types::INT32) {
- text = std::to_string(document[field_name].get<int32_t>());
+ auto val = document[field_name].get<int32_t>();
+ text = std::to_string(val);
} else if(the_field.type == field_types::INT64) {
- text = std::to_string(document[field_name].get<int64_t>());
+ auto val = document[field_name].get<int64_t>();
+ text = std::to_string(val);
} else if(the_field.type == field_types::FLOAT) {
- text = StringUtils::float_to_str(document[field_name].get<float>());
+ auto val = document[field_name].get<float>();
+ text = StringUtils::float_to_str(val);
} else if(the_field.type == field_types::BOOL) {
- text = std::to_string(document[field_name].get<bool>());
+ auto val = document[field_name].get<bool>();
+ text = std::to_string(val);
}
+ facet_hashes[text].emplace_back(record.seq_id);
+
tokenize_string(text, is_facet, the_field,
local_symbols_to_index, local_token_separators,
offsets);
@@ -311,10 +326,20 @@ void Index::compute_token_offsets_facets(index_record& record,
tokenize_string(document[field_name], is_facet, the_field,
local_symbols_to_index, local_token_separators,
offsets);
+ std::string val = document[field_name];
+ if(!val.empty()) {
+ facet_hashes[val].push_back(record.seq_id);
+ }
} else {
tokenize_string_array(document[field_name], is_facet, the_field,
local_symbols_to_index, local_token_separators,
offsets);
+
+ for(std::string val : document[field_name]) {
+ if(!val.empty()) {
+ facet_hashes[val].emplace_back(record.seq_id);
+ }
+ }
}
}
@@ -415,6 +440,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
for(size_t i = 0; i < batch_size; i++) {
index_record& index_rec = iter_batch[batch_start_index + i];
+ auto& facet_hashes = index_rec.facet_hashes;
try {
if(!index_rec.indexed.ok()) {
@@ -466,7 +492,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
docs_to_embed.push_back(&index_rec.doc);
}
- compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);
+ compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index, facet_hashes);
int64_t points = 0;
@@ -544,6 +570,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
+
cv_process.notify_one();
});
@@ -556,6 +583,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
}
std::unordered_set<std::string> found_fields;
+ std::map<std::string, std::vector<uint32_t>> facet_hashes;
for(size_t i = 0; i < iter_batch.size(); i++) {
auto& index_rec = iter_batch[i];
@@ -574,6 +602,11 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
for(const auto& kv: index_rec.doc.items()) {
found_fields.insert(kv.key());
}
+
+ for(const auto& kv : index_rec.facet_hashes) {
+ std::copy(kv.second.begin(), kv.second.end(),
+ std::back_inserter(facet_hashes[kv.first]));
+ }
}
num_queued = num_processed = 0;
@@ -592,7 +625,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b
const field& f = (field_name == "id") ?
field("id", field_types::STRING, false) : search_schema.at(field_name);
try {
- index->index_field_in_memory(f, iter_batch);
+ index->index_field_in_memory(f, iter_batch, facet_hashes);
} catch(std::exception& e) {
LOG(ERROR) << "Unhandled Typesense error: " << e.what();
for(auto& record: iter_batch) {
@@ -639,7 +672,8 @@ void Index::create_facet_hash_index(const field& facet_field) {
}
}
-void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {
+void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch,
+ std::map<std::string, std::vector<uint32_t>>& facet_hashes) {
// indexes a given field of all documents in the batch
if(afield.name == "id") {
@@ -704,7 +738,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
auto facet_index_it = facet_hash_index.find(afield.name);
if(facet_index_it != facet_hash_index.end()) {
facet_index = facet_index_it->second;
- }
+ }
for(const auto& record: iter_batch) {
if(!record.indexed.ok()) {
@@ -737,10 +771,19 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
} else if(afield.type == field_types::INT64_ARRAY) {
int64_t raw_val = document[afield.name][i].get<int64_t>();
value = std::to_string(raw_val);
- fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ auto it = count_index_map.find(value);
+ if(it == count_index_map.end()) {
+ count_index_map.emplace(value, ++count_index);
+ }
+ fhash = count_index_map.at(value);
} else if(afield.type == field_types::STRING_ARRAY) {
value = document[afield.name][i];
- fhash = facet_index_v4->insert(afield.name, value, seq_id, true);
+ auto it = count_index_map.find(value);
+ if(it == count_index_map.end()) {
+ count_index_map.emplace(value, ++count_index);
+ }
+ fhash = count_index_map.at(value);
+ facet_index_v4->insert(afield.name, value, facet_hashes[value], fhash);
} else if(afield.type == field_types::FLOAT_ARRAY) {
float raw_val = document[afield.name][i].get<float>();
fhash = reinterpret_cast<uint32_t&>(raw_val);
@@ -754,9 +797,11 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
}
}
- if(facet_threshold_count > FACET_INDEX_THRESHOLD) {
+ if(facet_index && facet_threshold_count > FACET_INDEX_THRESHOLD) {
facet_index->upsert(seq_id, std::move(fhashvalues));
fhashvalues.clear();
+ } else {
+ LOG(ERROR) << "facet_index was null while inserting for facet " << afield.name;
}
} else {
uint32_t fhash;
@@ -768,11 +813,20 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
else if(afield.type == field_types::INT64) {
int64_t raw_val = document[afield.name].get<int64_t>();
value = std::to_string(raw_val);
- fhash = facet_index_v4->insert(afield.name, value, seq_id);
+ auto it = count_index_map.find(value);
+ if(it == count_index_map.end()) {
+ count_index_map.emplace(value, ++count_index);
+ }
+ fhash = count_index_map.at(value);
}
else if(afield.type == field_types::STRING) {
value = document[afield.name];
- fhash = facet_index_v4->insert(afield.name, value, seq_id, true);
+ auto it = count_index_map.find(value);
+ if(it == count_index_map.end()) {
+ count_index_map.emplace(value, ++count_index);
+ }
+ fhash = count_index_map.at(value);
+ facet_index_v4->insert(afield.name, value, facet_hashes[value], fhash);
}
else if(afield.type == field_types::FLOAT) {
float raw_val = document[afield.name].get<float>();
@@ -783,9 +837,12 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>
fhash = (uint32_t)raw_val;
}
- if(facet_threshold_count > FACET_INDEX_THRESHOLD) {
+ if(facet_index && facet_threshold_count > FACET_INDEX_THRESHOLD) {
facet_index->upsert(seq_id, {fhash});
+ } else {
+ LOG(ERROR) << "facet_index was null while inserting for facet " << afield.name;
}
+
}
}
@@ -1156,12 +1213,14 @@ void Index::tokenize_string_array(const std::vector<std::string>& strings, bool
}
void Index::initialize_facet_indexes(const field& facet_field) {
- if(facet_field.is_string() || facet_field.is_int64()) {
+ if(facet_field.is_string()) {
facet_index_v4->initialize(facet_field.name);
}
-
- facet_hash_index.emplace(facet_field.name, new posting_list_t(256));
-
+
+ //LOG(INFO) << "emplacing facet " << facet_field.name;
+ if(facet_hash_index.find(facet_field.name) == facet_hash_index.end()) {
+ facet_hash_index.emplace(facet_field.name, new posting_list_t(256));
+ }
}
void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {