Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Prefix search to be used only for last token.

Kishore Nallan55535198a41b
diff --git a/src/art.cpp b/src/art.cpp
index c16a5afc..094c7ecc 100644
--- a/src/art.cpp
+++ b/src/art.cpp
@@ -1251,8 +1251,8 @@ static inline int fuzzy_search_state(const bool prefix, int key_index, bool last
return -1;
}
- if(key_len >= term_len) {
- // b) we might iterate past term_len to catch trailing typos
+ // b) we might iterate past term_len to catch trailing typos
+ if(key_len >= term_len && prefix) {
cost = cost_row[term_len];
if(cost >= min_cost && cost <= max_cost) {
return 1;
diff --git a/src/collection.cpp b/src/collection.cpp
index f031866c..bd8fbae3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1650,7 +1650,7 @@ void Collection::highlight_result(const field &search_field,
size_t match_offset_index = 0;
std::string raw_token;
- spp::sparse_hash_set<std::string> token_hits; // used to identify repeating tokens
+ std::set<std::string> token_hits; // used to identify repeating tokens
size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
// based on `highlight_affix_num_tokens`
diff --git a/src/index.cpp b/src/index.cpp
index becff94b..2f4d16ae 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -970,11 +970,6 @@ void Index::search_candidates(const uint8_t & field_id,
if(filter_ids_length != 0) {
id_found_in_filter = false;
- /*if(result_size != 0) {
- LOG(INFO) << size_t(field_id) << " - " << log_query.str() << ", result_size: " << result_size
- << ", popcount: " << (__builtin_popcount(token_bits) - 1);
- }*/
-
// e.g. [1, 3] vs [2, 3]
while(filter_ids_index < filter_ids_length && filter_ids[filter_ids_index] < id) {
diff --git a/test/art_test.cpp b/test/art_test.cpp
index 157b58dc..5b3419fd 100644
--- a/test/art_test.cpp
+++ b/test/art_test.cpp
@@ -636,6 +636,28 @@ TEST(ArtTest, test_art_fuzzy_search_single_leaf_prefix) {
ASSERT_TRUE(res == 0);
}
+TEST(ArtTest, test_art_fuzzy_search_single_leaf_non_prefix) {
+ art_tree t;
+ int res = art_tree_init(&t);
+ ASSERT_TRUE(res == 0);
+
+ const char* key = "spz005";
+ art_document doc = get_document((uint32_t) 1);
+ ASSERT_TRUE(NULL == art_insert(&t, (unsigned char*)key, strlen(key)+1, &doc));
+
+ std::string term = "spz";
+ std::vector<art_leaf*> leaves;
+ art_fuzzy_search(&t, (const unsigned char *)(term.c_str()), term.size()+1, 0, 1, 10, FREQUENCY, false, nullptr, 0, leaves);
+ ASSERT_EQ(0, leaves.size());
+
+ leaves.clear();
+ art_fuzzy_search(&t, (const unsigned char *)(term.c_str()), term.size(), 0, 1, 10, FREQUENCY, true, nullptr, 0, leaves);
+ ASSERT_EQ(1, leaves.size());
+
+ res = art_tree_destroy(&t);
+ ASSERT_TRUE(res == 0);
+}
+
TEST(ArtTest, test_art_fuzzy_search) {
art_tree t;
int res = art_tree_init(&t);
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 0779a08b..efce1dfa 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -546,11 +546,10 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
spp::sparse_hash_set<std::string>(), 10, "categories:cell ph").get();
ASSERT_EQ(1, results["facet_counts"].size());
- ASSERT_EQ(3, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
ASSERT_STREQ("Cell Phones", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("Cellophanes", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
- ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());
+ ASSERT_STREQ("Cell Phone Accessories", results["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());
// facet query longer than a token is correctly matched with typo tolerance
// also ensure that setting per_page = 0 works fine
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index c284d440..7b9d302c 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -800,3 +800,28 @@ TEST_F(CollectionSpecificTest, HighlightWithDropTokensAndPrefixSearch) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSpecificTest, PrefixSearchOnlyOnLastToken) {
+ std::vector<field> fields = {field("concat", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["concat"] = "SPZ005 SPACEPOLE Spz005 Space Pole Updated!!! Accessories Stands & Equipment Cabinets POS "
+ "Terminal Stand Spacepole 0 SPZ005";
+ doc1["points"] = 100;
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ auto results = coll1->search("spz space", {"concat"},
+ "", {}, {}, {1}, 10,
+ 1, FREQUENCY, {true},
+ 0, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "description", 20, {}, {}, {}, 0,
+ "<mark>", "</mark>").get();
+
+ ASSERT_EQ(0, results["hits"][0]["highlights"].size());
+
+ collectionManager.drop_collection("coll1");
+}