Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Merge branch '0.22.0-rc' into postings-refactor-integration

Kishore Nallan41c16fb7a737
diff --cc src/index.cpp
index 46376937,6f6f4b7d..3ddbb0f1
--- a/src/index.cpp
+++ b/src/index.cpp
@@@ -920,117 -934,92 +921,112 @@@ void Index::search_candidates(const uin
continue;
}
- uint32_t* result_ids = query_suggestion[0]->values->ids.uncompress();
+ // Prepare excluded document IDs that we can later remove from the result set
+ uint32_t* excluded_result_ids = nullptr;
+ size_t excluded_result_ids_size = ArrayUtils::or_scalar(exclude_token_ids, exclude_token_ids_size,
+ &curated_ids[0], curated_ids.size(), &excluded_result_ids);
- // intersect the document ids for each token to find docs that contain all the tokens (stored in `result_ids`)
- for(size_t i=1; i < query_suggestion.size(); i++) {
- uint32_t* out = nullptr;
- uint32_t* ids = query_suggestion[i]->values->ids.uncompress();
- result_size = ArrayUtils::and_scalar(ids, query_suggestion[i]->values->ids.getLength(), result_ids, result_size, &out);
- delete[] ids;
- delete[] result_ids;
- result_ids = out;
- }
+ std::vector<void*> posting_lists;
- if(result_size == 0) {
- delete[] result_ids;
- continue;
+ for(auto& query_leaf : query_suggestion) {
+ posting_lists.push_back(query_leaf->values);
}
- // Exclude document IDs associated with excluded tokens from the result set
- if(exclude_token_ids_size != 0) {
- uint32_t *excluded_result_ids = nullptr;
- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, exclude_token_ids, exclude_token_ids_size,
- &excluded_result_ids);
- delete[] result_ids;
- result_ids = excluded_result_ids;
- }
+ std::vector<posting_list_t::iterator_t> its;
+ posting_list_t::result_iter_state_t iter_state;
+ std::vector<uint32_t> result_id_vec;
- if(!curated_ids.empty()) {
- uint32_t *excluded_result_ids = nullptr;
- result_size = ArrayUtils::exclude_scalar(result_ids, result_size, &curated_ids[0],
- curated_ids.size(), &excluded_result_ids);
+ size_t excluded_result_ids_index = 0;
+ size_t filter_ids_index = 0;
- delete [] result_ids;
- result_ids = excluded_result_ids;
- }
+ posting_t::block_intersector_t intersector(posting_lists, 1000, iter_state);
+ bool has_more = true;
- //LOG(INFO) << "n: " << n;
- /*std::stringstream log_query;
- for(size_t i=0; i < query_suggestion.size(); i++) {
- log_query << query_suggestion[i]->key << " ";
- }*/
+ while(has_more) {
+ has_more = intersector.intersect();
+ posting_list_t::result_iter_state_t updated_iter_state;
+ size_t id_block_index = 0;
- if(filter_ids != nullptr) {
- // intersect once again with filter ids
- uint32_t* filtered_result_ids = nullptr;
- size_t filtered_results_size = ArrayUtils::and_scalar(filter_ids, filter_ids_length, result_ids,
- result_size, &filtered_result_ids);
+ for(size_t i = 0; i < iter_state.ids.size(); i++) {
+ uint32_t id = iter_state.ids[i];
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, filtered_result_ids,
- filtered_results_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ // decide if this result id should be excluded
+ if(excluded_result_ids_size != 0) {
+ while(excluded_result_ids_index < excluded_result_ids_size &&
+ excluded_result_ids[excluded_result_ids_index] < id) {
+ excluded_result_ids_index++;
+ }
- // go through each matching document id and calculate match score
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion,
- groups_processed, filtered_result_ids, filtered_results_size,
- group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);
+ if(excluded_result_ids_index < excluded_result_ids_size &&
+ id == excluded_result_ids[excluded_result_ids_index]) {
+ excluded_result_ids_index++;
+ continue;
+ }
+ }
- field_num_results += filtered_results_size;
+ bool id_found_in_filter = true;
- delete[] filtered_result_ids;
- delete[] result_ids;
- } else {
- uint32_t* new_all_result_ids = nullptr;
- all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
- result_size, &new_all_result_ids);
- delete [] *all_result_ids;
- *all_result_ids = new_all_result_ids;
+ // decide if this result be matched with filter results
+ if(filter_ids_length != 0) {
+ id_found_in_filter = false;
- /*if(result_size != 0) {
- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size
- << ", popcount: " << (__builtin_popcount(token_bits) - 1);
- }*/
+ /*if(result_size != 0) {
+ LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size
+ << ", popcount: " << (__builtin_popcount(token_bits) - 1);
+ }*/
- score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
- query_suggestion,
- groups_processed, result_ids, result_size, group_limit, group_by_fields, token_bits,
- query_tokens, prioritize_exact_match);
+ // e.g. [1, 3] vs [2, 3]
+
+ while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {
+ filter_ids_index++;
+ }
+
+ if(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] == id) {
+ filter_ids_index++;
+ id_found_in_filter = true;
+ }
+ }
- field_num_results += result_size;
+ if(id_found_in_filter) {
+ result_id_vec.push_back(id);
- delete[] result_ids;
+ updated_iter_state.num_lists = iter_state.num_lists;
+ updated_iter_state.ids.push_back(id);
+
+ for(size_t k = 0; k < iter_state.num_lists; k++) {
+ updated_iter_state.blocks.push_back(iter_state.blocks[id_block_index]);
+ updated_iter_state.indices.push_back(iter_state.indices[id_block_index++]);
+ }
+ }
+ }
+
+ std::vector<std::unordered_map<size_t, std::vector<token_positions_t>>> array_token_positions_vec;
+ posting_list_t::get_offsets(updated_iter_state, array_token_positions_vec);
+
+ score_results(sort_fields, (uint16_t) searched_queries.size(), field_id, total_cost, topster,
+ query_suggestion, groups_processed, array_token_positions_vec,
+ &updated_iter_state.ids[0], updated_iter_state.ids.size(),
+ group_limit, group_by_fields, token_bits, query_tokens, prioritize_exact_match);
}
+ if(result_id_vec.empty()) {
+ continue;
+ }
+
+ uint32_t* result_ids = &result_id_vec[0];
+ result_size = result_id_vec.size();
+
+ field_num_results += result_id_vec.size();
+
+ uint32_t* new_all_result_ids = nullptr;
+ all_result_ids_len = ArrayUtils::or_scalar(*all_result_ids, all_result_ids_len, result_ids,
+ result_size, &new_all_result_ids);
+ delete [] *all_result_ids;
+ *all_result_ids = new_all_result_ids;
+
+ delete [] excluded_result_ids;
+
searched_queries.push_back(actual_query_suggestion);
-
- //LOG(INFO) << "field_num_results: " << field_num_results << ", typo_tokens_threshold: " << typo_tokens_threshold;
- if(field_num_results >= typo_tokens_threshold) {
- break;
- }
}
}
@@@ -1665,17 -1707,24 +1661,23 @@@ void Index::search(const std::vector<qu
uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set
uint64_t total_typos = 0, total_distances = 0, min_typos = 1000;
- uint64_t verbatim_match_fields = 0; // query matching field verbatim
- uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens
+ uint64_t verbatim_match_fields = 0; // query matching field verbatim
+ uint64_t exact_match_fields = 0; // number of fields that contains all of query tokens
+ uint64_t max_weighted_tokens_match = 0; // weighted max number of tokens matched in a field
- uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)
+ uint64_t total_token_matches = 0; // total matches across fields (including fuzzy ones)
//LOG(INFO) << "Init pop count: " << __builtin_popcount(token_bits);
for(size_t i = 0; i < num_search_fields; i++) {
const auto field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
- size_t weight = search_fields[i].weight;
+ const size_t priority = search_fields[i].priority;
+ const size_t weight = search_fields[i].weight;
- //LOG(INFO) << "--- field index: " << i << ", weight: " << weight;
+ //LOG(INFO) << "--- field index: " << i << ", priority: " << priority;
-
+ // using `5` here because typo + prefix combo score range is: 0 - 5
+ // 0 1 2
+ // 0,1 2,3 4,5
+ int64_t MAX_SUM_TYPOS = 5 * field_query_tokens[i].q_include_tokens.size();
if(existing_field_kvs.count(field_id) != 0) {
// for existing field, we will simply sum field-wise weighted scores
diff --cc test/collection_specific_test.cpp
index fa086aa2,786211f1..f829cf13
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@@ -433,3 -546,70 +546,70 @@@ TEST_F(CollectionSpecificTest, DeleteOv
store->scan_fill(Collection::COLLECTION_SYNONYM_PREFIX, stored_values);
ASSERT_TRUE(stored_values.empty());
}
+
+ TEST_F(CollectionSpecificTest, SingleCharMatchFullFieldHighlight) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "Which of the following is a probable sign of infection?";
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ auto results = coll1->search("a 3-month", {"title"}, "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false}, 1,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
+ "title", 1).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("Which of the following is <mark>a</mark> probable sign of infection?",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ ASSERT_EQ("Which of the following is <mark>a</mark> probable sign of infection?",
+ results["hits"][0]["highlights"][0]["value"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+ }
+
+ TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("description", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "Foo bar baz";
+ doc1["description"] = "Share information with this device.";
+ doc1["points"] = 100;
+
+ nlohmann::json doc2;
+ doc2["id"] = "1";
+ doc2["title"] = "Foo Random";
+ doc2["description"] = "The Bar Fox";
+ doc2["points"] = 250;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());
+
+ auto results = coll1->search("foo bar", {"title", "description"}, "", {}, {}, {0}, 10,
+ 1, FREQUENCY, {false, false},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,
+ "<mark>", "</mark>", {4, 1}).get();
+
+ LOG(INFO) << results;
+
+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
-}
++}