Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Only last token should prefix search during drop tokens.
Kishore Nallanf45416be0c20
diff --git a/include/index.h b/include/index.hindex 1309abea..effe2bc8 100644--- a/include/index.h+++ b/include/index.h@@ -29,6 +29,11 @@ using array_mapped_facet_t = std::array<facet_map_t*, ARRAY_FACET_DIM>;struct token_t {size_t position;std::string value;+ bool prefix;++ token_t(size_t position, const std::string& value, bool prefix): position(position), value(value), prefix(prefix) {++ }};struct token_candidates {diff --git a/src/index.cpp b/src/index.cppindex 03faac11..19885cba 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1923,7 +1923,8 @@ bool Index::check_for_overrides(const token_ordering& token_order, const string&std::vector<token_t> window_tokens;std::set<std::string> window_tokens_set;for (size_t i = start_index; i < start_index + window_len; i++) {- window_tokens.push_back({i, tokens[i]});+ bool is_prefix = (i == (start_index + window_len - 1));+ window_tokens.emplace_back(i, tokens[i], is_prefix);window_tokens_set.emplace(tokens[i]);}@@ -2100,14 +2101,16 @@ void Index::search(std::vector<query_tokens_t>& field_query_tokens,for(size_t i = 0; i < num_search_fields; i++) {std::vector<token_t> q_include_pos_tokens;for(size_t j=0; j < field_query_tokens[i].q_include_tokens.size(); j++) {- q_include_pos_tokens.push_back({j, field_query_tokens[i].q_include_tokens[j]});+ bool is_prefix = (j == field_query_tokens[i].q_include_tokens.size()-1);+ q_include_pos_tokens.emplace_back(j, field_query_tokens[i].q_include_tokens[j], is_prefix);}std::vector<std::vector<token_t>> q_pos_synonyms;for(const auto& q_syn_vec: field_query_tokens[i].q_synonyms) {std::vector<token_t> q_pos_syn;for(size_t j=0; j < q_syn_vec.size(); j++) {- q_pos_syn.push_back({j, q_syn_vec[j]});+ bool is_prefix = (j == q_syn_vec.size()-1);+ q_pos_syn.emplace_back(j, q_syn_vec[j], is_prefix);}q_pos_synonyms.emplace_back(q_pos_syn);}@@ -2543,8 +2546,9 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&std::vector<token_t> search_tokens, qtokens;for (size_t qtoken_index = 0; qtoken_index < query_tokens.size(); qtoken_index++) {- search_tokens.emplace_back(token_t{qtoken_index, query_tokens[qtoken_index]});- qtokens.emplace_back(token_t{qtoken_index, query_tokens[qtoken_index]});+ bool is_prefix = (qtoken_index == query_tokens.size()-1);+ search_tokens.emplace_back(qtoken_index, query_tokens[qtoken_index], is_prefix);+ qtokens.emplace_back(qtoken_index, query_tokens[qtoken_index], is_prefix);}std::vector<std::vector<art_leaf*>> searched_queries;@@ -2893,14 +2897,14 @@ void Index::search_field(const uint8_t & field_id,const std::string token_cost_hash = token + std::to_string(costs[token_index]);std::vector<art_leaf*> leaves;- //LOG(INFO) << "Searching for field: " << field << ", token:" << token << " - cost: " << costs[token_index];+ const bool prefix_search = prefix && search_tokens[token_index].prefix;- const bool prefix_search = prefix && (token_index == search_tokens.size()-1);+ /*LOG(INFO) << "Searching for field: " << the_field.name << ", token:"+ << token << " - cost: " << costs[token_index] << ", prefix_search: " << prefix_search;*/if(token_cost_cache.count(token_cost_hash) != 0) {leaves = token_cost_cache[token_cost_hash];} else {- // prefix should apply only for last tokenconst size_t token_len = prefix_search ? (int) token.length() : (int) token.length() + 1;//auto begin = std::chrono::high_resolution_clock::now();@@ -3008,13 +3012,13 @@ void Index::search_field(const uint8_t & field_id,// drop from rightsize_t end_index = (query_tokens.size() - 1) - num_tokens_dropped;for(size_t i=0; i <= end_index; i++) {- truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});+ truncated_tokens.emplace_back(query_tokens[i].position, query_tokens[i].value, query_tokens[i].prefix);}} else {// drop from leftsize_t start_index = (num_tokens_dropped - mid_index);for(size_t i=start_index; i<query_tokens.size(); i++) {- truncated_tokens.push_back({query_tokens[i].position, query_tokens[i].value});+ truncated_tokens.emplace_back(query_tokens[i].position, query_tokens[i].value, query_tokens[i].prefix);}}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 8ae33b22..c148b659 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1711,8 +1711,6 @@ TEST_F(CollectionSpecificTest, RepeatingStringArrayTokens) {}TEST_F(CollectionSpecificTest, HighlightOnPrefixRegression) {- std::vector<std::string> tags;-// when the first document containing a token already cannot fit compact posting liststd::vector<field> fields = {field("title", field_types::STRING, false),};@@ -1727,8 +1725,46 @@ TEST_F(CollectionSpecificTest, HighlightOnPrefixRegression) {auto results = coll1->search("and", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(1, results["hits"].size());- LOG(INFO) << results;-collectionManager.drop_collection("coll1");}+TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeUsedForPrefixSearch) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "Dog Shoemaker";+ doc1["points"] = 100;++ nlohmann::json doc2;+ doc2["id"] = "1";+ doc2["title"] = "Shoe and Sock";+ doc2["points"] = 200;++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());+ ASSERT_TRUE(coll1->add(doc2.dump()).ok());++ auto results = coll1->search("shoe cat", {"title"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());++ results = coll1->search("cat shoe", {"title"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 20, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());+}