Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Extract out cross-field aggregation and scoring.
Kishore Nalland218db6487de
diff --git a/include/index.h b/include/index.hindex 4e25cffd..c1e2fbbd 100644--- a/include/index.h+++ b/include/index.h@@ -857,6 +857,11 @@ public:const std::vector<search_field_t>& the_fields, size_t& all_result_ids_len,uint32_t*& all_result_ids,spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const;++ void aggregate_and_score_fields(const std::vector<query_tokens_t>& field_query_tokens,+ const std::vector<search_field_t>& the_fields, Topster* topster,+ const size_t num_search_fields,+ spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const;};template<class T>diff --git a/src/index.cpp b/src/index.cppindex a41500b8..8f50e983 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1153,12 +1153,12 @@ void Index::search_candidates(const uint8_t & field_id, bool field_is_array,//LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;//LOG(INFO) << "n: " << n;- std::stringstream fullq;+ /*std::stringstream fullq;for(const auto& qleaf : actual_query_suggestion) {std::string qtok(reinterpret_cast<char*>(qleaf->key),qleaf->key_len - 1);fullq << qtok << " ";}- LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;+ LOG(INFO) << "field: " << size_t(field_id) << ", query: " << fullq.str() << ", total_cost: " << total_cost;*/// Prepare excluded document IDs that we can later remove from the result setuint32_t* excluded_result_ids = nullptr;@@ -2211,194 +2211,7 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,total_q_tokens += phrase.size();}*/- for(auto& seq_id_kvs: topster_ids) {- const uint64_t seq_id = seq_id_kvs.first;- auto& kvs = seq_id_kvs.second; // each `kv` can be from a different field-- std::sort(kvs.begin(), kvs.end(), Topster::is_greater);-- // kvs[0] will store query indices of the kv group (across fields)- kvs[0]->query_indices = new uint64_t[kvs.size() + 1];- kvs[0]->query_indices[0] = kvs.size();-- //LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index];-- // to calculate existing aggregate scores across best matching fields- spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;- for(size_t kv_i = 0; kv_i < kvs.size(); kv_i++) {- existing_field_kvs.emplace(kvs[kv_i]->field_id, kvs[kv_i]);- kvs[0]->query_indices[kv_i+1] = kvs[kv_i]->query_index;- /*LOG(INFO) << "kv_i: " << kv_i << ", kvs[kv_i]->query_index: " << kvs[kv_i]->query_index << ", "- << "searched_query: " << searched_queries[kvs[kv_i]->query_index][0];*/- }-- uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set- uint64_t total_typos = 0, total_distances = 0, min_typos = 1000;-- uint64_t verbatim_match_fields = 0; // field value *exactly* same as query tokens- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens- uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)-- //LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);-- for(size_t i = 0; i < num_search_fields; i++) {- const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);- const size_t priority = the_fields[i].priority;- const size_t weight = the_fields[i].weight;-- //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;-- if(existing_field_kvs.count(field_id) != 0) {- // for existing field, we will simply sum field-wise weighted scores- token_bits |= existing_field_kvs[field_id]->token_bits;- //LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);-- int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];-- uint64_t tokens_found = ((match_score >> 24) & 0xFF);- uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);- total_typos += (field_typos + 1) * priority;- total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;-- int64_t exact_match_score = match_score & 0xFF;- verbatim_match_fields += (weight * exact_match_score);-- uint64_t unique_tokens_found =- int64_t(__builtin_popcount(existing_field_kvs[field_id]->token_bits)) - 1;-- if(field_typos == 0 && unique_tokens_found == field_query_tokens[i].q_include_tokens.size()) {- exact_match_fields += weight;- }-- auto weighted_tokens_match = (tokens_found * weight);- if(weighted_tokens_match > max_weighted_tokens_match) {- max_weighted_tokens_match = weighted_tokens_match;- }-- if(field_typos < min_typos) {- min_typos = field_typos;- }-- total_token_matches += (weight * tokens_found);-- /*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))- << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * priority- << ", total dist: " << (((match_score & 0xFF)))- << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * priority;*/- continue;- }-- // compute approximate match score for this field from actual query- const std::string& field = the_fields[i].name;- size_t words_present = 0;-- // FIXME: must consider phrase tokens also- for(size_t token_index=0; token_index < field_query_tokens[i].q_include_tokens.size(); token_index++) {- const auto& token = field_query_tokens[i].q_include_tokens[token_index];- const art_leaf* leaf = (art_leaf *) art_search(search_index.at(field), (const unsigned char*) token.c_str(),- token.length()+1);-- if(!leaf) {- continue;- }-- if(!posting_t::contains(leaf->values, seq_id)) {- continue;- }-- token_bits |= 1UL << token_index; // sets nth bit- //LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);-- words_present += 1;-- /*if(!leaves.empty()) {- LOG(INFO) << "tok: " << leaves[0]->key;- }*/- }-- if(words_present != 0) {- uint64_t match_score = Match::get_match_score(words_present, 0, 0);-- uint64_t tokens_found = ((match_score >> 24) & 0xFF);- uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);- total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;- total_typos += (field_typos + 1) * priority;-- if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {- exact_match_fields += weight;- // not possible to calculate verbatim_match_fields accurately here, so we won't- }-- auto weighted_tokens_match = (tokens_found * weight);-- if(weighted_tokens_match > max_weighted_tokens_match) {- max_weighted_tokens_match = weighted_tokens_match;- }-- if(field_typos < min_typos) {- min_typos = field_typos;- }-- total_token_matches += (weight * tokens_found);- //LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);- }- }-- // num tokens present across fields including those containing typos- int64_t uniq_tokens_found = int64_t(__builtin_popcount(token_bits)) - 1;-- // verbtaim match should not consider dropped-token cases- if(uniq_tokens_found != field_query_tokens[0].q_include_tokens.size()) {- // also check for synonyms- bool found_verbatim_syn = false;- for(const auto& synonym: field_query_tokens[0].q_synonyms) {- if(uniq_tokens_found == synonym.size()) {- found_verbatim_syn = true;- break;- }- }-- if(!found_verbatim_syn) {- verbatim_match_fields = 0;- }- }-- // protect most significant byte from overflow, since topster uses int64_t- verbatim_match_fields = std::min<uint64_t>(INT8_MAX, verbatim_match_fields);-- exact_match_fields += verbatim_match_fields;- exact_match_fields = std::min<uint64_t>(255, exact_match_fields);- max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);- total_typos = std::min<uint64_t>(255, total_typos);- total_distances = std::min<uint64_t>(100, total_distances);-- uint64_t aggregated_score = (- (exact_match_fields << 48) | // number of fields that contain *all tokens* in the query- (max_weighted_tokens_match << 40) | // weighted max number of tokens matched in a field- (uniq_tokens_found << 32) | // number of unique tokens found across fields including typos- ((255 - min_typos) << 24) | // minimum typo cost across all fields- (total_token_matches << 16) | // total matches across fields including typos- ((255 - total_typos) << 8) | // total typos across fields (weighted)- ((100 - total_distances) << 0) // total distances across fields (weighted)- );-- //LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;-- /*LOG(INFO) << "seq id: " << seq_id- << ", verbatim_match_fields: " << verbatim_match_fields- << ", exact_match_fields: " << exact_match_fields- << ", max_weighted_tokens_match: " << max_weighted_tokens_match- << ", uniq_tokens_found: " << uniq_tokens_found- << ", min typo score: " << (255 - min_typos)- << ", total_token_matches: " << total_token_matches- << ", typo score: " << (255 - total_typos)- << ", distance score: " << (100 - total_distances)- << ", aggregated_score: " << aggregated_score << ", token_bits: " << token_bits;*/-- kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;- topster->add(kvs[0]);- }+ aggregate_and_score_fields(field_query_tokens, the_fields, topster, num_search_fields, topster_ids);/*auto timeMillis0 = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin0).count();@@ -2521,6 +2334,200 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,//LOG(INFO) << "Time taken for result calc: " << timeMillis << "ms";}+void Index::aggregate_and_score_fields(const std::vector<query_tokens_t>& field_query_tokens,+ const std::vector<search_field_t>& the_fields, Topster* topster,+ const size_t num_search_fields,+ spp::sparse_hash_map<uint64_t, std::vector<KV*>>& topster_ids) const {+ for(auto& seq_id_kvs: topster_ids) {+ const uint64_t seq_id = seq_id_kvs.first;+ auto& kvs = seq_id_kvs.second; // each `kv` can be from a different field++ std::sort(kvs.begin(), kvs.end(), Topster::is_greater);++ // kvs[0] will store query indices of the kv group (across fields)+ kvs[0]->query_indices = new uint64_t[kvs.size() + 1];+ kvs[0]->query_indices[0] = kvs.size();++ //LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index];++ // to calculate existing aggregate scores across best matching fields+ spp::sparse_hash_map<uint8_t, KV*> existing_field_kvs;+ for(size_t kv_i = 0; kv_i < kvs.size(); kv_i++) {+ existing_field_kvs.emplace(kvs[kv_i]->field_id, kvs[kv_i]);+ kvs[0]->query_indices[kv_i+1] = kvs[kv_i]->query_index;+ /*LOG(INFO) << "kv_i: " << kv_i << ", kvs[kv_i]->query_index: " << kvs[kv_i]->query_index << ", "+ << "searched_query: " << searched_queries[kvs[kv_i]->query_index][0];*/+ }++ uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set+ uint64_t total_typos = 0, total_distances = 0, min_typos = 1000;++ uint64_t verbatim_match_fields = 0; // field value *exactly* same as query tokens+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens+ uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)++ //LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);++ for(size_t i = 0; i < num_search_fields; i++) {+ const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);+ const size_t priority = the_fields[i].priority;+ const size_t weight = the_fields[i].weight;++ //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;++ if(existing_field_kvs.count(field_id) != 0) {+ // for existing field, we will simply sum field-wise weighted scores+ token_bits |= existing_field_kvs[field_id]->token_bits;+ //LOG(INFO) << "existing_field_kvs.count pop count: " << __builtin_popcount(token_bits);++ int64_t match_score = existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];++ uint64_t tokens_found = ((match_score >> 24) & 0xFF);+ uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);+ total_typos += (field_typos + 1) * priority;+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;++ int64_t exact_match_score = match_score & 0xFF;+ verbatim_match_fields += (weight * exact_match_score);++ uint64_t unique_tokens_found =+ int64_t(__builtin_popcount(existing_field_kvs[field_id]->token_bits)) - 1;++ if(field_typos == 0 && unique_tokens_found == field_query_tokens[i].q_include_tokens.size()) {+ exact_match_fields += weight;+ }++ auto weighted_tokens_match = (tokens_found * weight);+ if(weighted_tokens_match > max_weighted_tokens_match) {+ max_weighted_tokens_match = weighted_tokens_match;+ }++ if(field_typos < min_typos) {+ min_typos = field_typos;+ }++ total_token_matches += (weight * tokens_found);++ /*LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << (255 - ((match_score >> 8) & 0xFF))+ << ", weighted typos: " << std::max<uint64_t>((255 - ((match_score >> 8) & 0xFF)), 1) * priority+ << ", total dist: " << (((match_score & 0xFF)))+ << ", weighted dist: " << std::max<uint64_t>((100 - (match_score & 0xFF)), 1) * priority;*/+ continue;+ }++ // compute approximate match score for this field from actual query+ const std::string& field = the_fields[i].name;+ size_t words_present = 0;++ // FIXME: must consider phrase tokens also+ for(size_t token_index=0; token_index < field_query_tokens[i].q_include_tokens.size(); token_index++) {+ const auto& token = field_query_tokens[i].q_include_tokens[token_index];+ const art_leaf* leaf = (art_leaf *) art_search(search_index.at(field), (const unsigned char*) token.c_str(),+ token.length()+1);++ if(!leaf) {+ continue;+ }++ if(!posting_t::contains(leaf->values, seq_id)) {+ continue;+ }++ token_bits |= 1UL << token_index; // sets nth bit+ //LOG(INFO) << "token_index: " << token_index << ", pop count: " << __builtin_popcount(token_bits);++ words_present += 1;++ /*if(!leaves.empty()) {+ LOG(INFO) << "tok: " << leaves[0]->key;+ }*/+ }++ if(words_present != 0) {+ uint64_t match_score = Match::get_match_score(words_present, 0, 0);++ uint64_t tokens_found = ((match_score >> 24) & 0xFF);+ uint64_t field_typos = 255 - ((match_score >> 16) & 0xFF);+ total_distances += ((100 - ((match_score >> 8) & 0xFF)) + 1) * priority;+ total_typos += (field_typos + 1) * priority;++ if(field_typos == 0 && tokens_found == field_query_tokens[i].q_include_tokens.size()) {+ exact_match_fields += weight;+ // not possible to calculate verbatim_match_fields accurately here, so we won't+ }++ auto weighted_tokens_match = (tokens_found * weight);++ if(weighted_tokens_match > max_weighted_tokens_match) {+ max_weighted_tokens_match = weighted_tokens_match;+ }++ if(field_typos < min_typos) {+ min_typos = field_typos;+ }++ total_token_matches += (weight * tokens_found);+ //LOG(INFO) << "seq_id: " << seq_id << ", total_typos: " << ((match_score >> 8) & 0xFF);+ }+ }++ // num tokens present across fields including those containing typos+ int64_t uniq_tokens_found = int64_t(__builtin_popcount(token_bits)) - 1;++ // verbtaim match should not consider dropped-token cases+ if(uniq_tokens_found != field_query_tokens[0].q_include_tokens.size()) {+ // also check for synonyms+ bool found_verbatim_syn = false;+ for(const auto& synonym: field_query_tokens[0].q_synonyms) {+ if(uniq_tokens_found == synonym.size()) {+ found_verbatim_syn = true;+ break;+ }+ }++ if(!found_verbatim_syn) {+ verbatim_match_fields = 0;+ }+ }++ // protect most significant byte from overflow, since topster uses int64_t+ verbatim_match_fields = std::min<uint64_t>(INT8_MAX, verbatim_match_fields);++ exact_match_fields += verbatim_match_fields;+ exact_match_fields = std::min<uint64_t>(255, exact_match_fields);+ max_weighted_tokens_match = std::min<uint64_t>(255, max_weighted_tokens_match);+ total_typos = std::min<uint64_t>(255, total_typos);+ total_distances = std::min<uint64_t>(100, total_distances);++ uint64_t aggregated_score = (+ (exact_match_fields << 48) | // number of fields that contain *all tokens* in the query+ (max_weighted_tokens_match << 40) | // weighted max number of tokens matched in a field+ (uniq_tokens_found << 32) | // number of unique tokens found across fields including typos+ ((255 - min_typos) << 24) | // minimum typo cost across all fields+ (total_token_matches << 16) | // total matches across fields including typos+ ((255 - total_typos) << 8) | // total typos across fields (weighted)+ ((100 - total_distances) << 0) // total distances across fields (weighted)+ );++ //LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;++ /*LOG(INFO) << "seq id: " << seq_id+ << ", verbatim_match_fields: " << verbatim_match_fields+ << ", exact_match_fields: " << exact_match_fields+ << ", max_weighted_tokens_match: " << max_weighted_tokens_match+ << ", uniq_tokens_found: " << uniq_tokens_found+ << ", min typo score: " << (255 - min_typos)+ << ", total_token_matches: " << total_token_matches+ << ", typo score: " << (255 - total_typos)+ << ", distance score: " << (100 - total_distances)+ << ", aggregated_score: " << aggregated_score << ", token_bits: " << token_bits;*/++ kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;+ topster->add(kvs[0]);+ }+}+void Index::search_fields(const std::vector<filter>& filters,const std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,const std::vector<sort_by>& sort_fields_std, const std::vector<uint32_t>& num_typos,