Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Split tokens should require co-occurrence.
Kishore Nallan36382ed3f177
diff --git a/include/index.h b/include/index.hindex 32c8b5ad..ffec847c 100644--- a/include/index.h+++ b/include/index.h@@ -612,7 +612,7 @@ private:const std::string &field_name,nlohmann::json::iterator& array_iter, bool is_array, bool& array_ele_erased);- bool common_results_exist(std::vector<art_leaf*>& leaves) const;+ bool common_results_exist(std::vector<art_leaf*>& leaves, bool must_match_phrase) const;public:// for limiting number of results on multiple candidates / query rewritesdiff --git a/src/index.cpp b/src/index.cppindex 0c666115..29289d41 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -4411,7 +4411,7 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const strileaves.push_back(leaf);}- if(leaves.size() == qtokens.size() && common_results_exist(leaves)) {+ if(leaves.size() == qtokens.size() && common_results_exist(leaves, false)) {return ;}@@ -4457,7 +4457,7 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const strileaves.push_back(leaf);}- if(candidate_tokens.size() == leaves.size() && common_results_exist(leaves)) {+ if(candidate_tokens.size() == leaves.size() && common_results_exist(leaves, false)) {resolved_queries.push_back(candidate_tokens);return;}@@ -4487,7 +4487,7 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const strisecond_part.length() + 1));std::vector<art_leaf*> part_leaves = {first_leaf, second_leaf};- if(second_leaf != nullptr && common_results_exist(part_leaves)) {+ if(second_leaf != nullptr && common_results_exist(part_leaves, true)) {candidate_tokens.push_back(first_part);candidate_tokens.push_back(second_part);found_split = true;@@ -4506,9 +4506,9 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const strileaves.clear();- for(auto& token: candidate_tokens) {- art_leaf* leaf = static_cast<art_leaf*>(art_search(t, (const unsigned char*) token.c_str(),- token.length() + 1));+ for(auto& candidate_token: candidate_tokens) {+ art_leaf* leaf = static_cast<art_leaf*>(art_search(t, (const unsigned char*) candidate_token.c_str(),+ candidate_token.length() + 1));if(leaf == nullptr) {break;}@@ -4516,14 +4516,14 @@ void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const strileaves.push_back(leaf);}- if(candidate_tokens.size() == leaves.size() && common_results_exist(leaves)) {+ if(candidate_tokens.size() == leaves.size() && common_results_exist(leaves, false)) {resolved_queries.push_back(candidate_tokens);return;}}}-bool Index::common_results_exist(std::vector<art_leaf*>& leaves) const {+bool Index::common_results_exist(std::vector<art_leaf*>& leaves, bool must_match_phrase) const {std::vector<uint32_t> result_ids;std::vector<void*> leaf_vals;@@ -4532,7 +4532,23 @@ bool Index::common_results_exist(std::vector<art_leaf*>& leaves) const {}posting_t::intersect(leaf_vals, result_ids);- return !result_ids.empty();++ if(result_ids.empty()) {+ return false;+ }++ if(!must_match_phrase) {+ return !result_ids.empty();+ }++ uint32_t* phrase_ids = new uint32_t[result_ids.size()];+ size_t num_phrase_ids;++ posting_t::get_phrase_matches(leaf_vals, false, &result_ids[0], result_ids.size(),+ phrase_ids, num_phrase_ids);+ bool phrase_exists = (num_phrase_ids != 0);+ delete [] phrase_ids;+ return phrase_exists;}/*diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 0fd2dde7..e9148d84 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1916,7 +1916,7 @@ TEST_F(CollectionSpecificTest, DroppedTokensShouldNotBeUsedForPrefixSearch) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {+TEST_F(CollectionSpecificTest, SearchShouldSplitAndJoinTokens) {// when the first document containing a token already cannot fit compact posting liststd::vector<field> fields = {field("title", field_types::STRING, false),};@@ -1932,6 +1932,20 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {results = coll1->search("pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();ASSERT_EQ(1, results["hits"].size());+ results = coll1->search("nonstick pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(1, results["hits"].size());++ results = coll1->search("the pressurecooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(1, results["hits"].size());++ // splitting requires tokens to co-occur as a phrase in the dataset++ results = coll1->search("the pressureis", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(0, results["hits"].size());++ results = coll1->search("greatcooker", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();+ ASSERT_EQ(0, results["hits"].size());+results = coll1->search("t h e", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}, 0).get();ASSERT_EQ(1, results["hits"].size());