Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Merge branch '0.22.0-rc' into postings-refactor-integration
Kishore Nallan41c16fb7a737
diff --cc src/index.cppindex 46376937,6f6f4b7d..3ddbb0f1--- a/src/index.cpp+++ b/src/index.cpp@@@ -920,117 -934,92 +921,112 @@@ void Index::search_candidates(const uincontinue;}- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();+ // Prepare excluded document IDs that we can later remove from the result set+ uint32_t* excluded_result_ids = nullptr;+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)- for(size_t i=1; i < query_suggestion.size(); i++) {- uint32_t* out = nullptr;- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);- delete[] ids;- delete[] result_ids;- result_ids = out;- }+ std::vector<void*> posting_lists;- if(result_size == 0) {- delete[] result_ids;- continue;+ for(auto& query_leaf : query_suggestion) {+ posting_lists.push_back(query_leaf->values);}- // Exclude document IDs associated with excluded tokens from the result set- if(exclude_token_ids_size != 0) {- uint32_t *excluded_result_ids = nullptr;- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,- &excluded_result_ids);- delete[] result_ids;- result_ids = excluded_result_ids;- }+ std::vector<posting_list_t::iterator_t> its;+ posting_list_t::result_iter_state_t iter_state;+ std::vector<uint32_t> result_id_vec;- if(!curated_ids.empty()) {- uint32_t *excluded_result_ids = nullptr;- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],- curated_ids.size(), &excluded_result_ids);+ size_t excluded_result_ids_index = 0;+ size_t filter_ids_index = 0;- delete [] result_ids;- result_ids = excluded_result_ids;- }+ posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);+ bool has_more = true;- //LOG(INFO) << "n: " << n;- /*std::stringstream log_query;- for(size_t i=0; i < query_suggestion.size(); i++) {- log_query << query_suggestion[i]->key << " ";- }*/+ while(has_more) {+ has_more = intersector.intersect();+ posting_list_t::result_iter_state_t updated_iter_state;+ size_t id_block_index = 0;- if(filter_ids != nullptr) {- // intersect once again with filter ids- uint32_t* filtered_result_ids = nullptr;- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,- result_size, &filtered_result_ids);+ for(size_t i = 0; i < iter_state.ids.size(); i++) {+ uint32_t id = iter_state.ids[i];- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,- filtered_results_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ // decide if this result id should be excluded+ if(excluded_result_ids_size != 0) {+ while(excluded_result_ids_index < excluded_result_ids_size &&+ excluded_result_ids[excluded_result_ids_index] < id) {+ excluded_result_ids_index++;+ }- // go through each matching document id and calculate match score- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion,- groups_processed, filtered_result_ids, filtered_results_size,- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);+ if(excluded_result_ids_index < excluded_result_ids_size &&+ id == excluded_result_ids[excluded_result_ids_index]) {+ excluded_result_ids_index++;+ continue;+ }+ }- field_num_results += filtered_results_size;+ bool id_found_in_filter = true;- delete[] filtered_result_ids;- delete[] result_ids;- } else {- uint32_t* new_all_result_ids = nullptr;- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,- result_size, &new_all_result_ids);- delete [] *all_result_ids;- *all_result_ids = new_all_result_ids;+ // decide if this result be matched with filter results+ if(filter_ids_length != 0) {+ id_found_in_filter = false;- /*if(result_size != 0) {- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size- << ", popcount: " << (__builtin_popcount(token_bits) - 1);- }*/+ /*if(result_size != 0) {+ LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size+ << ", popcount: " << (__builtin_popcount(token_bits) - 1);+ }*/- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,- query_suggestion,- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits,- query_tokens, prioritize_exact_match);+ // e.g. [1, 3] vs [2, 3]++ while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {+ filter_ids_index++;+ }++ if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {+ filter_ids_index++;+ id_found_in_filter = true;+ }+ }- field_num_results += result_size;+ if(id_found_in_filter) {+ result_id_vec.push_back(id);- delete[] result_ids;+ updated_iter_state.num_lists = iter_state.num_lists;+ updated_iter_state.ids.push_back(id);++ for(size_t k = 0; k < iter_state.num_lists; k++) {+ updated_iter_state.blocks.push_back(iter_state.blocks[id_block_index]);+ updated_iter_state.indices.push_back(iter_state.indices[id_block_index++]);+ }+ }+ }++ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);++ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,+ query_suggestion, groups_processed, array_token_positions_vec,+ &updated_iter_state.ids[0], updated_iter_state.ids.size(),+ group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);}+ if(result_id_vec.empty()) {+ continue;+ }++ uint32_t* result_ids = &result_id_vec[0];+ result_size = result_id_vec.size();++ field_num_results += result_id_vec.size();++ uint32_t* new_all_result_ids = nullptr;+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,+ result_size, &new_all_result_ids);+ delete [] *all_result_ids;+ *all_result_ids = new_all_result_ids;++ delete [] excluded_result_ids;+searched_queries.push_back(actual_query_suggestion);-- //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;- if(field_num_results >= typo_tokens_threshold) {- break;- }}}@@@ -1665,17 -1707,24 +1661,23 @@@ void Index::search(const std::vector<quuint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit setuint64_t total_typos = 0, total_distances = 0, min_typos = 1000;- uint64_t verbatim_match_fields = 0; // query matching field verbatim- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens+ uint64_t verbatim_match_fields = 0; // query matching field verbatim+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens+ uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);for(size_t i = 0; i < num_search_fields; i++) {const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);- size_t weight = search_fields[i].weight;+ const size_t priority = search_fields[i].priority;+ const size_t weight = search_fields[i].weight;- //LOG(INFO) << "--- field index: " << i << ", weight: " << weight;+ //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;-+ // using `5` here because typo + prefix combo score range is: 0 - 5+ // 0 1 2+ // 0,1 2,3 4,5+ int64_t MAX_SUM_TYPOS = 5 * field_query_tokens[i].q_include_tokens.size();if(existing_field_kvs.count(field_id) != 0) {// for existing field, we will simply sum field-wise weighted scoresdiff --cc test/collection_specific_test.cppindex fa086aa2,786211f1..f829cf13--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@@ -433,3 -546,70 +546,70 @@@ TEST_F(CollectionSpecificTest, DeleteOvstore->scan_fill(Collection::COLLECTION_SYNONYM_PREFIX, stored_values);ASSERT_TRUE(stored_values.empty());}++ TEST_F(CollectionSpecificTest, SingleCharMatchFullFieldHighlight) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Which of the following is a probable sign of infection?";+ doc1["points"] = 100;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ auto results = coll1->search("a 3-month", {"title"}, "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false}, 1,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,+ "title", 1).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());++ ASSERT_EQ("Which of the following is <mark>a</mark> probable sign of infection?",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ ASSERT_EQ("Which of the following is <mark>a</mark> probable sign of infection?",+ results["hits"][0]["highlights"][0]["value"].get<std::string>());++ collectionManager.drop_collection("coll1");+ }++ TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("description", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Foo bar baz";+ doc1["description"] = "Share information with this device.";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["title"] = "Foo Random";+ doc2["description"] = "The Bar Fox";+ doc2["points"] = 250;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("foo bar", {"title", "description"}, "", {}, {}, {0}, 10,+ 1, FREQUENCY, {false, false},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<mark>", "</mark>", {4, 1}).get();++ LOG(INFO) << results;++ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());++ collectionManager.drop_collection("coll1");-}++}