Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
optimization with new index
krunal13134127c80aea51
diff --git a/include/facet_index.h b/include/facet_index.hindex c97d6097..124eaf72 100644--- a/include/facet_index.h+++ b/include/facet_index.h@@ -43,8 +43,7 @@ private:struct facet_index_counter {tsl::htrie_map<char, facet_index_struct> facet_index_map;- std::vector<count_list> counter_list;- bool is_migrated = false;+ std::vector<count_list*> counter_list;facet_index_counter() {facet_index_map.clear();@@ -57,7 +56,10 @@ private:}facet_index_map.clear();-++ for(auto val : counter_list) {+ delete val;+ }counter_list.clear();}};@@ -70,7 +72,8 @@ public:~facet_index_t();- uint32_t insert(const std::string& field, const std::string& value, uint32_t id, bool is_string = false);+ void insert(const std::string& field, const std::string& value,+ const std::vector<uint32_t>& ids, uint32_t index);void erase(const std::string& field);diff --git a/include/index.h b/include/index.hindex d2f725fc..4630cd63 100644--- a/include/index.h+++ b/include/index.h@@ -205,6 +205,8 @@ struct index_record {std::unordered_map<std::string, std::vector<uint32_t>>> field_index;int64_t points;+ std::map<std::string, std::vector<uint32_t>> facet_hashes;+Option<bool> indexed; // indicates if the indexing operation was a successDIRTY_VALUES dirty_values;@@ -308,6 +310,10 @@ private:facet_index_t* facet_index_v4 = nullptr;+ //for string and int64 facets insertions+ uint32_t count_index;++ std::unordered_map<std::string, uint32_t> count_index_map;// sort_field => (seq_id => value)spp::sparse_hash_map<std::string, spp::sparse_hash_map<uint32_t, int64_t>*> sort_index;@@ -608,7 +614,8 @@ public:static void compute_token_offsets_facets(index_record& record,const tsl::htrie_map<char, field>& search_schema,const std::vector<char>& local_token_separators,- const std::vector<char>& local_symbols_to_index);+ const std::vector<char>& local_symbols_to_index,+ std::map<std::string, std::vector<uint32_t>>& facet_hashes);static void scrub_reindex_doc(const tsl::htrie_map<char, field>& search_schema,nlohmann::json& update_doc, nlohmann::json& del_doc, const nlohmann::json& old_doc);@@ -674,7 +681,8 @@ public:const std::vector<char>& symbols_to_index,const bool do_validation);- void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch);+ void index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch,+ std::map<std::string, std::vector<uint32_t>>& facet_hashes);template<class T>void iterate_and_index_numerical_field(std::vector<index_record>& iter_batch, const field& afield, T func);diff --git a/src/facet_index.cpp b/src/facet_index.cppindex 88e8ec69..2719a134 100644--- a/src/facet_index.cpp+++ b/src/facet_index.cpp@@ -10,75 +10,63 @@ void facet_index_t::initialize(const std::string& field) {}}-uint32_t facet_index_t::insert(const std::string& field, const std::string& value,- uint32_t id, bool is_string) {+void facet_index_t::insert(const std::string& field, const std::string& value,+ const std::vector<uint32_t>& ids, uint32_t index) {const auto facet_field_map_it = facet_field_map.find(field);if(facet_field_map_it == facet_field_map.end()) {- return 0; //field is not initialized or dropped+ return; //field is not initialized or dropped}- uint32_t index = 0;-auto& facet_index_map = facet_field_map_it->second.facet_index_map;const auto sv = value.substr(0, 100);const auto it = facet_index_map.find(sv);if(it == facet_index_map.end()) {- index = ++count_index;facet_index_struct fis{};fis.index = index;- if(is_string) {- fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(1, {id}));- }+ fis.id_list_ptr = SET_COMPACT_IDS(compact_id_list_t::create(ids.size(), ids));facet_index_map.emplace(sv, fis);} else {- if(is_string) {- auto ids = it->id_list_ptr;- if (!ids_t::contains(ids, id)) {- ids_t::upsert(ids, id);- facet_index_map[sv].id_list_ptr = ids;+ auto ids_ptr = it->id_list_ptr;+ for(const auto& id : ids) {+ if (!ids_t::contains(ids_ptr, id)) {+ ids_t::upsert(ids_ptr, id);+ facet_index_map[sv].id_list_ptr = ids_ptr;}}- index = facet_index_map[sv].index;}- if(is_string) {- const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);- //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;- auto& counter_list = facet_field_map_it->second.counter_list;+ const auto facet_count = ids_t::num_ids(facet_index_map.at(sv).id_list_ptr);+ //LOG(INFO) << "Facet count in facet " << sv << " : " << facet_count;+ auto& counter_list = facet_field_map_it->second.counter_list;- if(counter_list.empty()) {- counter_list.emplace_back(sv, facet_count, index);- } else {- count_list node(sv, facet_count, index);- auto ind = 0;-- for(; ind < counter_list.size(); ++ind) {- if(counter_list[ind].index == index) {- counter_list[ind].count = facet_count;-- if(ind > 1) {- auto curr = ind;- while (curr && (counter_list[curr-1].count < counter_list[curr].count)) {- count_list temp = counter_list[curr-1];- counter_list[curr-1] = counter_list[curr];- counter_list[curr] = temp;- --curr;- }+ if(counter_list.empty()) {+ count_list* node = new count_list(sv, facet_count, index);+ counter_list.emplace_back(node);+ } else {+ auto ind = 0;++ for(; ind < counter_list.size(); ++ind) {+ if(counter_list[ind]->index == index) {+ counter_list[ind]->count = facet_count;+ if(ind > 1) {+ auto curr = ind;+ while (curr && (counter_list[curr-1]->count < counter_list[curr]->count)) {+ std::swap(counter_list[curr-1], counter_list[curr]);+ --curr;}- break;}- }- if(ind == counter_list.size()) {- // LOG (INFO) << "inserting at last facet " << node.facet_value- // << " with count " << node.count;- counter_list.emplace_back(node);+ break;}}+ if(ind == counter_list.size()) {+ // LOG (INFO) << "inserting at last facet " << node.facet_value+ // << " with count " << node.count;+ count_list* node = new count_list(sv, facet_count, index);+ counter_list.emplace_back(node);+ }}-- return index;}bool facet_index_t::contains(const std::string& field) {@@ -137,7 +125,7 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* result}const auto facet_index_map = facet_field_it->second.facet_index_map;- const auto counter_list = facet_field_it->second.counter_list;+ const auto& counter_list = facet_field_it->second.counter_list;// LOG (INFO) << "facet_index_map size " << facet_index_map.size()// << " , counter_list size " << counter_list.size();@@ -150,9 +138,9 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* resultuint32_t count = 0;if(is_wildcard_no_filter_query) {- count = counter_list_it.count;+ count = counter_list_it->count;} else {- auto ids = facet_index_map.at(counter_list_it.facet_value).id_list_ptr;+ auto ids = facet_index_map.at(counter_list_it->facet_value).id_list_ptr;ids_t::uncompress(ids, id_list);const auto ids_len = id_list.size();for(int i = 0; i < result_ids_len; ++i) {@@ -165,7 +153,7 @@ size_t facet_index_t::intersect(const std::string& field, const uint32_t* result}if(count) {- found[counter_list_it.facet_value] = count;+ found[counter_list_it->facet_value] = count;if(found.size() == max_facets) {break;}diff --git a/src/index.cpp b/src/index.cppindex 881d7a64..a6cee7c3 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -246,7 +246,8 @@ float Index::int64_t_to_float(int64_t n) {void Index::compute_token_offsets_facets(index_record& record,const tsl::htrie_map<char, field>& search_schema,const std::vector<char>& local_token_separators,- const std::vector<char>& local_symbols_to_index) {+ const std::vector<char>& local_symbols_to_index,+ std::map<std::string, std::vector<uint32_t>>& facet_hashes) {const auto& document = record.doc;@@ -267,19 +268,27 @@ void Index::compute_token_offsets_facets(index_record& record,if(the_field.type == field_types::INT32_ARRAY) {for(int32_t value: document[field_name]){- strings.push_back(std::to_string(value));+ auto str = std::to_string(value);+ facet_hashes[str].emplace_back(record.seq_id);+ strings.emplace_back(std::move(str));}} else if(the_field.type == field_types::INT64_ARRAY) {for(int64_t value: document[field_name]){- strings.push_back(std::to_string(value));+ auto str = std::to_string(value);+ facet_hashes[str].emplace_back(record.seq_id);+ strings.emplace_back(std::move(str));}} else if(the_field.type == field_types::FLOAT_ARRAY) {for(float value: document[field_name]){- strings.push_back(StringUtils::float_to_str(value));+ auto str = StringUtils::float_to_str(value);+ facet_hashes[str].emplace_back(record.seq_id);+ strings.emplace_back(std::move(str));}} else if(the_field.type == field_types::BOOL_ARRAY) {for(bool value: document[field_name]){- strings.push_back(std::to_string(value));+ auto str = std::to_string(value);+ facet_hashes[str].emplace_back(record.seq_id);+ strings.emplace_back(std::move(str));}}@@ -290,15 +299,21 @@ void Index::compute_token_offsets_facets(index_record& record,std::string text;if(the_field.type == field_types::INT32) {- text = std::to_string(document[field_name].get<int32_t>());+ auto val = document[field_name].get<int32_t>();+ text = std::to_string(val);} else if(the_field.type == field_types::INT64) {- text = std::to_string(document[field_name].get<int64_t>());+ auto val = document[field_name].get<int64_t>();+ text = std::to_string(val);} else if(the_field.type == field_types::FLOAT) {- text = StringUtils::float_to_str(document[field_name].get<float>());+ auto val = document[field_name].get<float>();+ text = StringUtils::float_to_str(val);} else if(the_field.type == field_types::BOOL) {- text = std::to_string(document[field_name].get<bool>());+ auto val = document[field_name].get<bool>();+ text = std::to_string(val);}+ facet_hashes[text].emplace_back(record.seq_id);+tokenize_string(text, is_facet, the_field,local_symbols_to_index, local_token_separators,offsets);@@ -311,10 +326,20 @@ void Index::compute_token_offsets_facets(index_record& record,tokenize_string(document[field_name], is_facet, the_field,local_symbols_to_index, local_token_separators,offsets);+ std::string val = document[field_name];+ if(!val.empty()) {+ facet_hashes[val].push_back(record.seq_id);+ }} else {tokenize_string_array(document[field_name], is_facet, the_field,local_symbols_to_index, local_token_separators,offsets);++ for(std::string val : document[field_name]) {+ if(!val.empty()) {+ facet_hashes[val].emplace_back(record.seq_id);+ }+ }}}@@ -415,6 +440,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itefor(size_t i = 0; i < batch_size; i++) {index_record& index_rec = iter_batch[batch_start_index + i];+ auto& facet_hashes = index_rec.facet_hashes;try {if(!index_rec.indexed.ok()) {@@ -466,7 +492,7 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& itedocs_to_embed.push_back(&index_rec.doc);}- compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);+ compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index, facet_hashes);int64_t points = 0;@@ -544,6 +570,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bstd::unique_lock<std::mutex> lock(m_process);num_processed++;+cv_process.notify_one();});@@ -556,6 +583,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_b}std::unordered_set<std::string> found_fields;+ std::map<std::string, std::vector<uint32_t>> facet_hashes;for(size_t i = 0; i < iter_batch.size(); i++) {auto& index_rec = iter_batch[i];@@ -574,6 +602,11 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bfor(const auto& kv: index_rec.doc.items()) {found_fields.insert(kv.key());}++ for(const auto& kv : index_rec.facet_hashes) {+ std::copy(kv.second.begin(), kv.second.end(),+ std::back_inserter(facet_hashes[kv.first]));+ }}num_queued = num_processed = 0;@@ -592,7 +625,7 @@ size_t Index::batch_memory_index(Index *index, std::vector<index_record>& iter_bconst field& f = (field_name == "id") ?field("id", field_types::STRING, false) : search_schema.at(field_name);try {- index->index_field_in_memory(f, iter_batch);+ index->index_field_in_memory(f, iter_batch, facet_hashes);} catch(std::exception& e) {LOG(ERROR) << "Unhandled Typesense error: " << e.what();for(auto& record: iter_batch) {@@ -639,7 +672,8 @@ void Index::create_facet_hash_index(const field& facet_field) {}}-void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch) {+void Index::index_field_in_memory(const field& afield, std::vector<index_record>& iter_batch,+ std::map<std::string, std::vector<uint32_t>>& facet_hashes) {// indexes a given field of all documents in the batchif(afield.name == "id") {@@ -704,7 +738,7 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>auto facet_index_it = facet_hash_index.find(afield.name);if(facet_index_it != facet_hash_index.end()) {facet_index = facet_index_it->second;- }+ }for(const auto& record: iter_batch) {if(!record.indexed.ok()) {@@ -737,10 +771,19 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>} else if(afield.type == field_types::INT64_ARRAY) {int64_t raw_val = document[afield.name][i].get<int64_t>();value = std::to_string(raw_val);- fhash = facet_index_v4->insert(afield.name, value, seq_id);+ auto it = count_index_map.find(value);+ if(it == count_index_map.end()) {+ count_index_map.emplace(value, ++count_index);+ }+ fhash = count_index_map.at(value);} else if(afield.type == field_types::STRING_ARRAY) {value = document[afield.name][i];- fhash = facet_index_v4->insert(afield.name, value, seq_id, true);+ auto it = count_index_map.find(value);+ if(it == count_index_map.end()) {+ count_index_map.emplace(value, ++count_index);+ }+ fhash = count_index_map.at(value);+ facet_index_v4->insert(afield.name, value, facet_hashes[value], fhash);} else if(afield.type == field_types::FLOAT_ARRAY) {float raw_val = document[afield.name][i].get<float>();fhash = reinterpret_cast<uint32_t&>(raw_val);@@ -754,9 +797,11 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>}}- if(facet_threshold_count > FACET_INDEX_THRESHOLD) {+ if(facet_index && facet_threshold_count > FACET_INDEX_THRESHOLD) {facet_index->upsert(seq_id, std::move(fhashvalues));fhashvalues.clear();+ } else {+ LOG(ERROR) << "facet_index was null while inserting for facet " << afield.name;}} else {uint32_t fhash;@@ -768,11 +813,20 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>else if(afield.type == field_types::INT64) {int64_t raw_val = document[afield.name].get<int64_t>();value = std::to_string(raw_val);- fhash = facet_index_v4->insert(afield.name, value, seq_id);+ auto it = count_index_map.find(value);+ if(it == count_index_map.end()) {+ count_index_map.emplace(value, ++count_index);+ }+ fhash = count_index_map.at(value);}else if(afield.type == field_types::STRING) {value = document[afield.name];- fhash = facet_index_v4->insert(afield.name, value, seq_id, true);+ auto it = count_index_map.find(value);+ if(it == count_index_map.end()) {+ count_index_map.emplace(value, ++count_index);+ }+ fhash = count_index_map.at(value);+ facet_index_v4->insert(afield.name, value, facet_hashes[value], fhash);}else if(afield.type == field_types::FLOAT) {float raw_val = document[afield.name].get<float>();@@ -783,9 +837,12 @@ void Index::index_field_in_memory(const field& afield, std::vector<index_record>fhash = (uint32_t)raw_val;}- if(facet_threshold_count > FACET_INDEX_THRESHOLD) {+ if(facet_index && facet_threshold_count > FACET_INDEX_THRESHOLD) {facet_index->upsert(seq_id, {fhash});+ } else {+ LOG(ERROR) << "facet_index was null while inserting for facet " << afield.name;}+}}@@ -1156,12 +1213,14 @@ void Index::tokenize_string_array(const std::vector<std::string>& strings, bool}void Index::initialize_facet_indexes(const field& facet_field) {- if(facet_field.is_string() || facet_field.is_int64()) {+ if(facet_field.is_string()) {facet_index_v4->initialize(facet_field.name);}-- facet_hash_index.emplace(facet_field.name, new posting_list_t(256));-++ //LOG(INFO) << "emplacing facet " << facet_field.name;+ if(facet_hash_index.find(facet_field.name) == facet_hash_index.end()) {+ facet_hash_index.emplace(facet_field.name, new posting_list_t(256));+ }}void Index::compute_facet_stats(facet &a_facet, const std::string& raw_value, const std::string & field_type) {